From 8d3bbb913029989eeb61d194ebb70852df38c3d8 Mon Sep 17 00:00:00 2001 From: Erik Mellegard Date: Fri, 27 Sep 2024 13:08:00 +0200 Subject: [PATCH 1/5] Use UCDs instead of json for ESA Sky --- astroquery/esasky/core.py | 215 +++++++++++++++++++------------------- 1 file changed, 106 insertions(+), 109 deletions(-) diff --git a/astroquery/esasky/core.py b/astroquery/esasky/core.py index a86dd03c1c..af1b499451 100644 --- a/astroquery/esasky/core.py +++ b/astroquery/esasky/core.py @@ -48,7 +48,7 @@ class ESASkyClass(BaseQuery): __OBSERVATIONS_STRING = "observations" __SPECTRA_STRING = "spectra" __MISSION_STRING = "mission" - __TAP_TABLE_STRING = "tapTable" + __TAP_TABLE_STRING = "table_name" __TAP_NAME_STRING = "tapName" __TAP_RA_COLUMN_STRING = "tapRaColumn" __TAP_DEC_COLUMN_STRING = "tapDecColumn" @@ -208,29 +208,25 @@ def list_maps(self): """ Get a list of the mission names of the available observations in ESASky """ - return self._json_object_field_to_list( - self._get_observation_json(), self.__MISSION_STRING) + return self._get_observation_info().keys() def list_catalogs(self): """ Get a list of the mission names of the available catalogs in ESASky """ - return self._json_object_field_to_list( - self._get_catalogs_json(), self.__MISSION_STRING) + return self._get_catalogs_info().keys() def list_spectra(self): """ Get a list of the mission names of the available spectra in ESASky """ - return self._json_object_field_to_list( - self._get_spectra_json(), self.__MISSION_STRING) + return self._get_spectra_info().keys() def list_sso(self): """ Get a list of the mission names of the available observations with SSO crossmatch in ESASky """ - return self._json_object_field_to_list( - self._get_sso_json(), self.__MISSION_STRING) + return self._get_sso_info().keys() def query_object_maps(self, position, missions=__ALL_STRING, get_query_payload=False, cache=True, row_limit=DEFAULT_ROW_LIMIT, verbose=False): @@ -537,7 +533,7 @@ def query_sso(self, sso_name, *, sso_type="ALL", missions=__ALL_STRING, row_limi sanitized_row_limit = self._sanitize_input_row_limit(row_limit) sso = sso[0] - sso_json = self._get_sso_json() + sso_info = self._get_sso_info() query_result = {} @@ -547,9 +543,8 @@ def query_sso(self, sso_name, *, sso_type="ALL", missions=__ALL_STRING, row_limi if sanitized_row_limit > 0: top = "TOP {row_limit} ".format(row_limit=sanitized_row_limit) for name in sanitized_missions: - data_table = self._find_mission_tap_table_name(sso_json, name) - mission_json = self._find_mission_parameters_in_json(data_table, sso_json) - x_match_table = mission_json['ssoXMatchTapTable'] + data_table = sso_info[name]['table_name'] + x_match_table = self._x_match_table(data_table) query = 'SELECT {top}* FROM {data_table} AS a JOIN {x_match_table} AS b ' \ 'ON a.observation_oid = b.observation_oid JOIN sso.ssoid AS c ' \ 'ON b.sso_oid = c.sso_oid WHERE c.{sso_db_identifier} = \'{sso_name}\' ' \ @@ -630,7 +625,7 @@ def get_images_sso(self, *, sso_name=None, sso_type="ALL", table_list=None, miss if sso_name is None and table_list is None: raise ValueError("An input is required for either sso_name or table.") - sanitized_missions = [m.lower() for m in self._sanitize_input_sso_mission(missions)] + sanitized_missions = self._sanitize_input_sso_mission(missions) sso_name = self._sanitize_input_sso_name(sso_name) sso_type = self._sanitize_input_sso_type(sso_type) if table_list is None: @@ -641,12 +636,14 @@ def get_images_sso(self, *, sso_name=None, sso_type="ALL", table_list=None, miss maps = dict() - json = self._get_sso_json() + descriptors = self._get_sso_info() + for query_mission in map_query_result.keys(): - if query_mission.lower() in sanitized_missions: + if query_mission in sanitized_missions: + table = self._table_for_mission(query_mission, descriptors) maps[query_mission] = ( self._get_maps_for_mission(map_query_result[query_mission], query_mission, download_dir, cache, - json, verbose=verbose)) + table, verbose=verbose)) if len(map_query_result) > 0 and all([maps[mission].count(None) == len(maps[mission]) for mission in maps]): @@ -716,8 +713,9 @@ def query_region_maps(self, position, radius, missions=__ALL_STRING, get_query_p sesame_database.set('simbad') coordinates = commons.parse_coordinates(position) - self._store_query_result(query_result=query_result, names=sanitized_missions, json=self._get_observation_json(), - coordinates=coordinates, radius=sanitized_radius, row_limit=sanitized_row_limit, + self._store_query_result(query_result=query_result, names=sanitized_missions, + descriptors=self._get_observation_info(), coordinates=coordinates, + radius=sanitized_radius, row_limit=sanitized_row_limit, get_query_payload=get_query_payload, cache=cache, verbose=verbose) if get_query_payload: return query_result @@ -783,8 +781,9 @@ def query_region_catalogs(self, position, radius, catalogs=__ALL_STRING, row_lim query_result = {} - self._store_query_result(query_result=query_result, names=sanitized_catalogs, json=self._get_catalogs_json(), - coordinates=coordinates, radius=sanitized_radius, row_limit=sanitized_row_limit, + self._store_query_result(query_result=query_result, names=sanitized_catalogs, + descriptors=self._get_catalogs_info(), coordinates=coordinates, + radius=sanitized_radius, row_limit=sanitized_row_limit, get_query_payload=get_query_payload, cache=cache, verbose=verbose) if get_query_payload: @@ -851,8 +850,9 @@ def query_region_spectra(self, position, radius, missions=__ALL_STRING, row_limi sesame_database.set('simbad') coordinates = commons.parse_coordinates(position) - self._store_query_result(query_result=query_result, names=sanitized_missions, json=self._get_spectra_json(), - coordinates=coordinates, radius=sanitized_radius, row_limit=sanitized_row_limit, + self._store_query_result(query_result=query_result, names=sanitized_missions, + descriptors=self._get_spectra_info(), coordinates=coordinates, + radius=sanitized_radius, row_limit=sanitized_row_limit, get_query_payload=get_query_payload, cache=cache, verbose=verbose) if get_query_payload: @@ -907,8 +907,9 @@ def query_ids_maps(self, observation_ids, *, missions=__ALL_STRING, row_limit=DE sanitized_row_limit = self._sanitize_input_row_limit(row_limit) query_result = {} - self._store_query_result(query_result=query_result, names=sanitized_missions, json=self._get_observation_json(), - row_limit=sanitized_row_limit, get_query_payload=get_query_payload, cache=cache, + self._store_query_result(query_result=query_result, names=sanitized_missions, + descriptors=self._get_observation_info(), row_limit=sanitized_row_limit, + get_query_payload=get_query_payload, cache=cache, ids=sanitized_observation_ids, verbose=verbose) if get_query_payload: @@ -963,8 +964,9 @@ def query_ids_catalogs(self, source_ids, *, catalogs=__ALL_STRING, row_limit=DEF sanitized_source_ids = self._sanitize_input_ids(source_ids) query_result = {} - self._store_query_result(query_result=query_result, names=sanitized_catalogs, json=self._get_catalogs_json(), - row_limit=sanitized_row_limit, get_query_payload=get_query_payload, cache=cache, + self._store_query_result(query_result=query_result, names=sanitized_catalogs, + descriptors=self._get_catalogs_info(), row_limit=sanitized_row_limit, + get_query_payload=get_query_payload, cache=cache, ids=sanitized_source_ids, verbose=verbose) if get_query_payload: @@ -1019,8 +1021,9 @@ def query_ids_spectra(self, observation_ids, *, missions=__ALL_STRING, row_limit sanitized_row_limit = self._sanitize_input_row_limit(row_limit) query_result = {} - self._store_query_result(query_result=query_result, names=sanitized_missions, json=self._get_spectra_json(), - row_limit=sanitized_row_limit, get_query_payload=get_query_payload, cache=cache, + self._store_query_result(query_result=query_result, names=sanitized_missions, + descriptors=self._get_spectra_info(), row_limit=sanitized_row_limit, + get_query_payload=get_query_payload, cache=cache, ids=sanitized_observation_ids, verbose=verbose) if get_query_payload: @@ -1081,14 +1084,15 @@ def get_maps(self, query_table_list, *, missions=__ALL_STRING, sanitized_missions = [m.lower() for m in self._sanitize_input_mission(missions)] maps = dict() - json = self._get_observation_json() + descriptors = self._get_observation_info() for query_mission in sanitized_query_table_list.keys(): if query_mission.lower() in sanitized_missions: + table = self._table_for_mission(query_mission, descriptors) maps[query_mission] = ( self._get_maps_for_mission(sanitized_query_table_list[query_mission], query_mission, download_dir, - cache, json, verbose=verbose)) + cache, table, verbose=verbose)) if all([maps[mission].count(None) == len(maps[mission]) for mission in maps]): @@ -1182,10 +1186,11 @@ def get_images(self, *, position=None, observation_ids=None, radius=__ZERO_ARCMI verbose=verbose) maps = dict() - json = self._get_observation_json() + descriptors = self._get_observation_info() for query_mission in map_query_result.keys(): + table = self._table_for_mission(query_mission, descriptors) maps[query_mission] = ( - self._get_maps_for_mission(map_query_result[query_mission], query_mission, download_dir, cache, json, + self._get_maps_for_mission(map_query_result[query_mission], query_mission, download_dir, cache, table, verbose=verbose)) if all([maps[mission].count(None) == len(maps[mission]) @@ -1272,11 +1277,12 @@ def get_spectra(self, *, position=None, observation_ids=None, radius=__ZERO_ARCM spectra_query_result = self.query_ids_spectra(missions=sanitized_missions, observation_ids=sanitized_observation_ids, get_query_payload=False, cache=cache, verbose=verbose) - json = self._get_spectra_json() + descriptors = self._get_spectra_info() for query_mission in spectra_query_result.keys(): + table = self._table_for_mission(query_mission, descriptors) spectra[query_mission] = ( self._get_maps_for_mission(spectra_query_result[query_mission], query_mission, download_dir, cache, - json, is_spectra=True, verbose=verbose)) + table, is_spectra=True, verbose=verbose)) if len(spectra_query_result) > 0: log.info("Spectra available at {}".format(os.path.abspath(download_dir))) @@ -1338,14 +1344,15 @@ def get_spectra_from_table(self, query_table_list, missions=__ALL_STRING, sanitized_missions = [m.lower() for m in self._sanitize_input_spectra(missions)] spectra = dict() - json = self._get_spectra_json() + descriptors = self._get_spectra_info() for query_mission in sanitized_query_table_list.keys(): if query_mission.lower() in sanitized_missions: + table = self._table_for_mission(query_mission, descriptors) spectra[query_mission] = ( self._get_maps_for_mission(sanitized_query_table_list[query_mission], query_mission, download_dir, - cache, json, is_spectra=True, verbose=verbose)) + cache, table, is_spectra=True, verbose=verbose)) if len(sanitized_query_table_list) > 0: log.info("Spectra available at {}.".format(os.path.abspath(download_dir))) @@ -1455,7 +1462,7 @@ def _get_db_sso_identifier(self, sso_type): return "sso_id" return "sso_name" - def _get_maps_for_mission(self, maps_table, mission, download_dir, cache, json, is_spectra=False, verbose=False): + def _get_maps_for_mission(self, maps_table, mission, download_dir, cache, table, is_spectra=False, verbose=False): if is_spectra and mission.lower() == self.__HERSCHEL_STRING: maps = dict() else: @@ -1486,8 +1493,8 @@ def _get_maps_for_mission(self, maps_table, mission, download_dir, cache, json, if isinstance(observation_id, bytes): observation_id = observation_id.decode('utf-8') else: - observation_id = \ - maps_table[self._get_json_data_for_mission(json, mission)["uniqueIdentifierField"]][index] + identifier = self._get_unique_identifier(table) + observation_id = maps_table[identifier][index] if isinstance(observation_id, bytes): observation_id = observation_id.decode('utf-8') log.debug("Downloading Observation ID: {} from {}".format(observation_id, product_url)) @@ -1697,16 +1704,15 @@ def _extract_file_name_from_url(self, product_url): start_index = product_url.rindex("/") + 1 return product_url[start_index:] - def _query(self, name, json, verbose=False, **kwargs): - table_tap_name = self._find_mission_tap_table_name(json, name) + def _query(self, name, descriptors, verbose=False, **kwargs): if 'ids' in kwargs: query = self._build_id_query(ids=kwargs.get('ids'), row_limit=kwargs.get('row_limit'), - json=self._find_mission_parameters_in_json(table_tap_name, json)) + descriptor=descriptors[name]) else: query = self._build_region_query(coordinates=kwargs.get('coordinates'), radius=kwargs.get('radius'), row_limit=kwargs.get('row_limit'), - json=self._find_mission_parameters_in_json(table_tap_name, json)) + descriptor=descriptors[name]) if 'get_query_payload' in kwargs and kwargs.get('get_query_payload'): return self._create_request_payload(query) @@ -1721,7 +1727,7 @@ def _query(self, name, json, verbose=False, **kwargs): return self.query(query, output_format="votable", verbose=verbose) - def _build_region_query(self, coordinates, radius, row_limit, json): + def _build_region_query(self, coordinates, radius, row_limit, descriptor): ra = coordinates.transform_to('icrs').ra.deg dec = coordinates.transform_to('icrs').dec.deg radius_deg = Angle(radius).to_value(u.deg) @@ -1734,12 +1740,12 @@ def _build_region_query(self, coordinates, radius, row_limit, json): select_query = "".join([select_query, "* "]) - tap_ra_column = json[self.__TAP_RA_COLUMN_STRING] - tap_dec_column = json[self.__TAP_DEC_COLUMN_STRING] + tap_ra_column = descriptor['ra'] + tap_dec_column = descriptor['dec'] - from_query = " FROM {}".format(json[self.__TAP_TABLE_STRING]) + from_query = " FROM {}".format(descriptor['table_name']) if radius_deg == 0: - if json[self.__USE_INTERSECT_STRING]: + if descriptor['intersect_polygon_query']: where_query = (" WHERE 1=INTERSECTS(CIRCLE('ICRS', {}, {}, {}), fov)". format(ra, dec, self.__MIN_RADIUS_CATALOG_DEG)) else: @@ -1749,19 +1755,18 @@ def _build_region_query(self, coordinates, radius, row_limit, json): dec, self.__MIN_RADIUS_CATALOG_DEG)) else: - if json[self.__USE_INTERSECT_STRING]: + if descriptor['intersect_polygon_query']: where_query = (" WHERE 1=INTERSECTS(CIRCLE('ICRS', {}, {}, {}), fov)". format(ra, dec, radius_deg)) else: where_query = (" WHERE 1=CONTAINS(POINT('ICRS', {}, {}), CIRCLE('ICRS', {}, {}, {}))". format(tap_ra_column, tap_dec_column, ra, dec, radius_deg)) - query = "".join([select_query, from_query, where_query]) return query - def _build_id_query(self, ids, row_limit, json): + def _build_id_query(self, ids, row_limit, descriptor): select_query = "SELECT " if row_limit > 0: select_query = "".join([select_query, "TOP {} ".format(row_limit)]) @@ -1770,16 +1775,17 @@ def _build_id_query(self, ids, row_limit, json): select_query = "".join([select_query, "* "]) - from_query = " FROM {}".format(json[self.__TAP_TABLE_STRING]) - id_column = json["uniqueIdentifierField"] - if "observations" in json["tapTable"] or "spectra" in json["tapTable"]: + table_name = descriptor[self.__TAP_TABLE_STRING] + from_query = " FROM {}".format(table_name) + id_column = self._get_unique_identifier(table_name) + if "observations" in table_name or "spectra" in table_name: if id_column in ("observation_oid", "plane_id"): id_column = "observation_id" if id_column == "designation": id_column = "obsid" data_type = None - for column in self.get_columns(table_name=json['tapTable'], only_names=False): + for column in self.get_columns(table_name=descriptor['table_name'], only_names=False): if column.name == id_column: data_type = column.data_type @@ -1787,8 +1793,8 @@ def _build_id_query(self, ids, row_limit, json): if data_type in self._NUMBER_DATA_TYPES: valid_ids = [int(obs_id) for obs_id in ids if obs_id.isdigit()] if not valid_ids: - raise ValueError(f"Could not construct query for mission {json['mission']}. Database column type is " - "a number, while none of the input id's could be interpreted as numbers.") + raise ValueError(f"Could not construct query for mission {descriptor['mission']}. Database column " + "type is a number, while none of the input id's could be interpreted as numbers.") return "" observation_ids_query_list = ", ".join(repr(id) for id in valid_ids) @@ -1799,61 +1805,52 @@ def _build_id_query(self, ids, row_limit, json): return query - def _store_query_result(self, query_result, names, json, verbose=False, **kwargs): + def _store_query_result(self, query_result, names, descriptors, verbose=False, **kwargs): for name in names: - table = self._query(name=name, json=json, verbose=verbose, **kwargs) + table = self._query(name=name, descriptors=descriptors, verbose=verbose, **kwargs) if len(table) > 0: query_result[name.upper()] = table - def _find_mission_parameters_in_json(self, mission_tap_name, json): - for mission in json: - if mission[self.__TAP_TABLE_STRING] == mission_tap_name: - return mission - raise ValueError("Input tap name {} not available.".format(mission_tap_name)) - - def _find_mission_tap_table_name(self, json, mission_name): - for index in range(len(json)): - if json[index][self.__MISSION_STRING].lower() == mission_name.lower(): - return json[index][self.__TAP_TABLE_STRING] - - raise ValueError("Input {} not available.".format(mission_name)) - - def _get_observation_json(self): - return self._fetch_and_parse_json(self.__OBSERVATIONS_STRING) - - def _get_catalogs_json(self): - return self._fetch_and_parse_json(self.__CATALOGS_STRING) - - def _get_spectra_json(self): - return self._fetch_and_parse_json(self.__SPECTRA_STRING) - - def _get_sso_json(self): - return self._fetch_and_parse_json("sso") - - def _fetch_and_parse_json(self, object_name): - url = self.URLbase + "/" + object_name - response = self._request( - 'GET', - url, - cache=False, - headers=self._get_header()) - - response.raise_for_status() - - string_response = response.content.decode('utf-8') - json_response = json.loads(string_response) - return json_response["descriptors"] - - def _json_object_field_to_list(self, json, field_name): - response_list = [] - for index in range(len(json)): - response_list.append(json[index][field_name]) - return response_list - - def _get_json_data_for_mission(self, json, mission): - for index in range(len(json)): - if json[index][self.__MISSION_STRING].lower() == mission.lower(): - return json[index] + def _get_observation_info(self): + return self._get_descriptors(category='observations') + + def _get_catalogs_info(self): + return self._get_descriptors(category='catalogues') + + def _get_spectra_info(self): + return self._get_descriptors(category='spectra') + + def _get_sso_info(self): + return self._get_descriptors(category='sso') + + def _get_descriptors(self, category): + query = """select d.*, ra.column_name as ra, dec.column_name as dec from descriptors d + join tap_schema.columns ra on d.table_name = ra.table_name + join tap_schema.columns dec on d.table_name = dec.table_name + where d.category = '{}' and ra.ucd = 'pos.eq.ra;meta.main' and dec.ucd = 'pos.eq.dec;meta.main' + """.format(category) + return {d['mission']: {a: d[a] for a in d.keys()} for d in self.query(query)} + + def _table_for_mission(self, mission, descriptors): + tables = {d['mission'].lower(): d['table_name'] for d in descriptors.values()} + return tables[mission.lower()] + + def _get_unique_identifier(self, table_name): + rows = self.query("select * from tap_schema.columns where table_name='{}'".format(table_name)) + for row in rows: + if 'meta.id' in row.get('ucd') and 'meta.main' in row.get('ucd'): + return row.get('column_name') + for row in rows: + if row.get('utype').lower() == 'DataID.observationID'.lower(): + return row.get('column_name') + for row in rows: + if row.get('utype').lower() == 'obscore:DataID.Creatordid'.lower(): + return row.get('column_name') + + def _x_match_table(self, data_table): + query = """SELECT xmatch_table from tap_descriptors.sso_extras + where descriptor_table_name='{}'""".format(data_table) + return self.query(query)[0]['xmatch_table'] def _create_request_payload(self, query): return {'REQUEST': 'doQuery', 'LANG': 'ADQL', 'FORMAT': 'VOTABLE', From 9cb0b92bd1134b22be29f255131eed67cd48d502 Mon Sep 17 00:00:00 2001 From: Erik Mellegard Date: Fri, 27 Sep 2024 14:08:53 +0200 Subject: [PATCH 2/5] Remove unused constants --- astroquery/esasky/core.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/astroquery/esasky/core.py b/astroquery/esasky/core.py index af1b499451..cc402a3c77 100644 --- a/astroquery/esasky/core.py +++ b/astroquery/esasky/core.py @@ -44,24 +44,13 @@ class ESASkyClass(BaseQuery): __FTZ_STRING = ".FTZ" __TAR_STRING = ".tar" __ALL_STRING = "all" - __CATALOGS_STRING = "catalogs" - __OBSERVATIONS_STRING = "observations" - __SPECTRA_STRING = "spectra" - __MISSION_STRING = "mission" __TAP_TABLE_STRING = "table_name" - __TAP_NAME_STRING = "tapName" - __TAP_RA_COLUMN_STRING = "tapRaColumn" - __TAP_DEC_COLUMN_STRING = "tapDecColumn" - __METADATA_STRING = "metadata" __PRODUCT_URL_STRING = "product_url" __EROSITA_PRODUCT_URL_STRING = "prod_url" __ACCESS_URL_STRING = "access_url" - __USE_INTERSECT_STRING = "useIntersectPolygonInsteadOfContainsPoint" __ZERO_ARCMIN_STRING = "0 arcmin" __MIN_RADIUS_CATALOG_DEG = Angle(5 * u.arcsec).to_value(u.deg) - __HERSCHEL_STRING = 'herschel' - __HST_STRING = 'hst' __HERSCHEL_FILTERS = { 'psw': '250', From 20fff6676f02863324165e556240b3489758b39b Mon Sep 17 00:00:00 2001 From: Erik Mellegard Date: Wed, 9 Oct 2024 12:33:51 +0200 Subject: [PATCH 3/5] Make list of missions lists instead of dict_keys --- astroquery/esasky/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astroquery/esasky/core.py b/astroquery/esasky/core.py index cc402a3c77..1fa687c5f7 100644 --- a/astroquery/esasky/core.py +++ b/astroquery/esasky/core.py @@ -197,25 +197,25 @@ def list_maps(self): """ Get a list of the mission names of the available observations in ESASky """ - return self._get_observation_info().keys() + return list(self._get_observation_info().keys()) def list_catalogs(self): """ Get a list of the mission names of the available catalogs in ESASky """ - return self._get_catalogs_info().keys() + return list(self._get_catalogs_info().keys()) def list_spectra(self): """ Get a list of the mission names of the available spectra in ESASky """ - return self._get_spectra_info().keys() + return list(self._get_spectra_info().keys()) def list_sso(self): """ Get a list of the mission names of the available observations with SSO crossmatch in ESASky """ - return self._get_sso_info().keys() + return list(self._get_sso_info().keys()) def query_object_maps(self, position, missions=__ALL_STRING, get_query_payload=False, cache=True, row_limit=DEFAULT_ROW_LIMIT, verbose=False): From fa816c73f7393a3d0e43577f41e2744d86ca51b1 Mon Sep 17 00:00:00 2001 From: Erik Mellegard Date: Wed, 9 Oct 2024 12:34:42 +0200 Subject: [PATCH 4/5] Make all mission strings in esasky upper case again --- astroquery/esasky/core.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/astroquery/esasky/core.py b/astroquery/esasky/core.py index 1fa687c5f7..34755789b3 100644 --- a/astroquery/esasky/core.py +++ b/astroquery/esasky/core.py @@ -1358,34 +1358,34 @@ def _sanitize_input_radius(self, radius): def _sanitize_input_mission(self, missions): if isinstance(missions, list): - return missions + return [m.upper() for m in missions] if isinstance(missions, str): if missions.lower() == self.__ALL_STRING: return self.list_maps() else: - return [missions] + return [missions.upper()] raise ValueError("Mission must be either a string or a list of " "missions") def _sanitize_input_spectra(self, spectra): if isinstance(spectra, list): - return spectra + return [s.upper() for s in spectra] if isinstance(spectra, str): if spectra.lower() == self.__ALL_STRING: return self.list_spectra() else: - return [spectra] + return [spectra.upper()] raise ValueError("Spectra must be either a string or a list of " "Spectra") def _sanitize_input_catalogs(self, catalogs): if isinstance(catalogs, list): - return catalogs + return [c.upper() for c in catalogs] if isinstance(catalogs, str): if catalogs.lower() == self.__ALL_STRING: return self.list_catalogs() else: - return [catalogs] + return [catalogs.upper()] raise ValueError("Catalog must be either a string or a list of " "catalogs") @@ -1798,7 +1798,7 @@ def _store_query_result(self, query_result, names, descriptors, verbose=False, * for name in names: table = self._query(name=name, descriptors=descriptors, verbose=verbose, **kwargs) if len(table) > 0: - query_result[name.upper()] = table + query_result[name] = table def _get_observation_info(self): return self._get_descriptors(category='observations') @@ -1818,7 +1818,7 @@ def _get_descriptors(self, category): join tap_schema.columns dec on d.table_name = dec.table_name where d.category = '{}' and ra.ucd = 'pos.eq.ra;meta.main' and dec.ucd = 'pos.eq.dec;meta.main' """.format(category) - return {d['mission']: {a: d[a] for a in d.keys()} for d in self.query(query)} + return {d['mission'].upper(): {a: d[a] for a in d.keys()} for d in self.query(query)} def _table_for_mission(self, mission, descriptors): tables = {d['mission'].lower(): d['table_name'] for d in descriptors.values()} From 9f63fe913ceb52d65bc06efe25976bb78189a73c Mon Sep 17 00:00:00 2001 From: Erik Mellegard Date: Wed, 9 Oct 2024 12:37:13 +0200 Subject: [PATCH 5/5] Fix esasky docs: rearrange output order --- docs/esasky/esasky.rst | 93 ++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/docs/esasky/esasky.rst b/docs/esasky/esasky.rst index 47d24a9b1f..dcd746580d 100644 --- a/docs/esasky/esasky.rst +++ b/docs/esasky/esasky.rst @@ -32,12 +32,13 @@ If you know the names of all the available catalogs you can use :meth:`~astroque >>> from astroquery.esasky import ESASky >>> catalog_list = ESASky.list_catalogs() >>> print(catalog_list) - ['LAMOST_LRS', 'LAMOST_MRS', 'AllWise', 'Spitzer', 'AKARI-IRC-SC', 'TwoMASS', 'INTEGRAL', 'CHANDRA-SC2', - 'XMM-EPIC-STACK', 'XMM-EPIC', 'XMM-OM', 'XMM-SLEW', 'Tycho-2', 'Gaia-DR3', 'Hipparcos-2', 'HSC', - 'Herschel-HPPSC-070', 'Herschel-HPPSC-100', 'Herschel-HPPSC-160', 'Herschel-SPSC-250', 'Herschel-SPSC-350', - 'Herschel-SPSC-500', 'Planck-PGCC', 'Planck-PCCS2E-HFI', 'Planck-PCCS2-HFI', 'Planck-PCCS2-LFI', 'Planck-PSZ2', - 'Icecube', 'Fermi_4FGL-DR2', 'Fermi_3FHL', 'Fermi_4LAC-DR2', '2WHSP', '2RXS', 'OU_Blazars', - 'eROSITA-eFEDS-hard', 'eROSITA-eFEDS-main', 'eROSITA-etaCha-hard', 'eROSITA-etaCha-main'] + ['TYCHO-2', '2RXS', 'INTEGRAL', 'GAIA-DR3', 'XMM-EPIC', 'OU_BLAZARS', 'XMM-SLEW', 'HIPPARCOS-2', + 'HERSCHEL-SPSC-500', 'AKARI-IRC-SC', 'HERSCHEL-HPPSC-070', 'HERSCHEL-HPPSC-100', 'HERSCHEL-HPPSC-160', + 'HERSCHEL-SPSC-250', 'HERSCHEL-SPSC-350', 'PLANCK-PCCS2E-HFI', 'PLANCK-PGCC', 'PLANCK-PCCS2-HFI', + 'PLANCK-PCCS2-LFI', 'PLANCK-PSZ2', 'CHANDRA-SC2', 'ALLWISE', 'TWOMASS', 'EROSITA-EFEDS-MAIN', + 'EROSITA-EFEDS-HARD', 'XMM-OM', 'XMM-EPIC-STACK', 'FERMI_4FGL-DR2', 'FERMI_3FHL', 'EROSITA-ETACHA-MAIN', + 'SWIFT-2SXPS', 'ICECUBE', 'FERMI_4LAC-DR2', 'EROSITA-ETACHA-HARD', 'HSC', 'PLATO ASPIC1.1', '2WHSP', + 'GAIA-FPR', 'EROSITA-ERASS-MAIN', 'EROSITA-ERASS-HARD', 'GLADE+', 'LAMOST_MRS', 'LAMOST_LRS'] Get the available maps mission names ------------------------------------ @@ -48,8 +49,8 @@ If you know the names of all the available maps missions you can use :meth:`~ast >>> maps_list = ESASky.list_maps() >>> print(maps_list) - ['INTEGRAL', 'XMM', 'Chandra', 'SUZAKU', 'XMM-OM-OPTICAL', 'XMM-OM-UV', 'HST-UV', 'HST-OPTICAL', 'HST-IR', - 'ISO-IR', 'Herschel', 'AKARI', 'JWST-MID-IR', 'JWST-NEAR-IR', 'Spitzer', 'ALMA', 'eROSITA'] + ['ALMA', 'ISO-IR', 'SPITZER', 'AKARI', 'HST-IR', 'HST-UV', 'HST-OPTICAL', 'EROSITA', 'INTEGRAL', + 'SUZAKU', 'HERSCHEL', 'JWST-MID-IR', 'JWST-NEAR-IR', 'XMM', 'XMM-OM-UV', 'XMM-OM-OPTICAL', 'CHANDRA'] Get the available spectra mission names --------------------------------------- @@ -60,8 +61,8 @@ If you know the names of all the available spectra you can use :meth:`~astroquer >>> spectra_list = ESASky.list_spectra() >>> print(spectra_list) - ['XMM-NEWTON', 'Chandra', 'IUE', 'HST-UV', 'HST-OPTICAL', 'JWST-MID-IR', 'JWST-NEAR-IR', 'HST-IR', 'ISO-IR', - 'Herschel', 'LAMOST_LRS', 'LAMOST_MRS', 'CHEOPS'] + ['HERSCHEL', 'CHANDRA', 'IUE', 'ISO-IR', 'CHEOPS', 'XMM-NEWTON', 'JWST-MID-IR', 'JWST-NEAR-IR', + 'HST-OPTICAL', 'HST-UV', 'HST-IR', 'LAMOST_MRS', 'LAMOST_LRS'] Get the available SSO mission names ----------------------------------- @@ -73,7 +74,7 @@ If you know the names of all the available missions with SSO cross match data, y >>> sso_list = ESASky.list_sso() >>> print(sso_list) - ['Herschel', 'HST', 'XMM', 'XMM-OM'] + ['XMM-OM', 'HST', 'HERSCHEL', 'XMM'] Query an object @@ -115,16 +116,18 @@ To see the result: .. doctest-remote-data:: >>> print(result) - TableList with 9 tables: - '0:ALLWISE' with 25 column(s) and 1 row(s) - '1:TWOMASS' with 14 column(s) and 3 row(s) - '2:CHANDRA-SC2' with 41 column(s) and 9 row(s) - '3:XMM-EPIC-STACK' with 347 column(s) and 1 row(s) - '4:XMM-EPIC' with 223 column(s) and 12 row(s) - '5:XMM-OM' with 122 column(s) and 5 row(s) - '6:HSC' with 27 column(s) and 230 row(s) - '7:HERSCHEL-HPPSC-070' with 21 column(s) and 1 row(s) - '8:HERSCHEL-HPPSC-100' with 21 column(s) and 1 row(s) + TableList with 11 tables: + '0:XMM-EPIC' with 223 column(s) and 15 row(s) + '1:HERSCHEL-HPPSC-070' with 21 column(s) and 1 row(s) + '2:HERSCHEL-HPPSC-100' with 21 column(s) and 1 row(s) + '3:CHANDRA-SC2' with 41 column(s) and 9 row(s) + '4:ALLWISE' with 25 column(s) and 1 row(s) + '5:TWOMASS' with 14 column(s) and 3 row(s) + '6:XMM-OM' with 122 column(s) and 7 row(s) + '7:XMM-EPIC-STACK' with 161 column(s) and 15 row(s) + '8:SWIFT-2SXPS' with 232 column(s) and 1 row(s) + '9:HSC' with 27 column(s) and 230 row(s) + '10:GLADE+' with 40 column(s) and 1 row(s) All the results are returned as a `~astroquery.utils.TableList` object. This is a container for `~astropy.table.Table` objects. It is basically an extension to `~collections.OrderedDict` for storing a `~astropy.table.Table` against its @@ -195,27 +198,29 @@ To see the result: .. doctest-remote-data:: >>> print(result) - TableList with 20 tables: - '0:LAMOST_LRS' with 43 column(s) and 37 row(s) - '1:ALLWISE' with 25 column(s) and 1762 row(s) - '2:SPITZER' with 146 column(s) and 1082 row(s) - '3:AKARI-IRC-SC' with 29 column(s) and 1 row(s) - '4:TWOMASS' with 14 column(s) and 188 row(s) - '5:CHANDRA-SC2' with 41 column(s) and 430 row(s) - '6:XMM-EPIC-STACK' with 347 column(s) and 225 row(s) - '7:XMM-EPIC' with 223 column(s) and 941 row(s) - '8:XMM-OM' with 122 column(s) and 4849 row(s) - '9:XMM-SLEW' with 106 column(s) and 2 row(s) - '10:GAIA-DR3' with 153 column(s) and 932 row(s) - '11:HSC' with 27 column(s) and 10000 row(s) - '12:HERSCHEL-HPPSC-070' with 21 column(s) and 93 row(s) - '13:HERSCHEL-HPPSC-100' with 21 column(s) and 122 row(s) - '14:HERSCHEL-HPPSC-160' with 21 column(s) and 93 row(s) - '15:HERSCHEL-SPSC-250' with 36 column(s) and 59 row(s) - '16:HERSCHEL-SPSC-350' with 36 column(s) and 24 row(s) - '17:HERSCHEL-SPSC-500' with 36 column(s) and 7 row(s) - '18:PLANCK-PCCS2-HFI' with 9 column(s) and 8 row(s) - '19:2RXS' with 306 column(s) and 2 row(s) + TableList with 22 tables: + '0:2RXS' with 306 column(s) and 2 row(s) + '1:GAIA-DR3' with 153 column(s) and 932 row(s) + '2:XMM-EPIC' with 223 column(s) and 1467 row(s) + '3:XMM-SLEW' with 106 column(s) and 2 row(s) + '4:HERSCHEL-SPSC-500' with 36 column(s) and 7 row(s) + '5:AKARI-IRC-SC' with 29 column(s) and 1 row(s) + '6:HERSCHEL-HPPSC-070' with 21 column(s) and 93 row(s) + '7:HERSCHEL-HPPSC-100' with 21 column(s) and 122 row(s) + '8:HERSCHEL-HPPSC-160' with 21 column(s) and 93 row(s) + '9:HERSCHEL-SPSC-250' with 36 column(s) and 59 row(s) + '10:HERSCHEL-SPSC-350' with 36 column(s) and 24 row(s) + '11:PLANCK-PCCS2-HFI' with 9 column(s) and 8 row(s) + '12:CHANDRA-SC2' with 41 column(s) and 430 row(s) + '13:ALLWISE' with 25 column(s) and 1762 row(s) + '14:TWOMASS' with 14 column(s) and 188 row(s) + '15:XMM-OM' with 122 column(s) and 7026 row(s) + '16:XMM-EPIC-STACK' with 161 column(s) and 4185 row(s) + '17:SWIFT-2SXPS' with 232 column(s) and 120 row(s) + '18:HSC' with 27 column(s) and 10000 row(s) + '19:PLATO ASPIC1.1' with 70 column(s) and 3 row(s) + '20:GLADE+' with 40 column(s) and 51 row(s) + '21:LAMOST_LRS' with 40 column(s) and 47 row(s) You can use, :meth:`~astroquery.esasky.ESASkyClass.query_region_maps` and :meth:`~astroquery.esasky.ESASkyClass.query_region_maps` with the same parameters. To execute the same command as above @@ -427,7 +432,7 @@ You can see the available missions with: >>> from astroquery.esasky import ESASky >>> ESASky.list_sso() - ['Herschel', 'HST', 'XMM', 'XMM-OM'] + ['XMM-OM', 'HST', 'HERSCHEL', 'XMM'] Other parameters and the return value are structured in the same manner as the other query methods. @@ -451,8 +456,8 @@ Or download everything on an SSO by something like this: >>> from astroquery.esasky import ESASky >>> images=ESASky.get_images_sso(sso_name="2017 RN65") - INFO: Starting download of HERSCHEL data. (1 files) [astroquery.esasky.core] INFO: Starting download of HST data. (1 files) [astroquery.esasky.core] + INFO: Starting download of HERSCHEL data. (1 files) [astroquery.esasky.core] INFO: Starting download of XMM data. (1 files) [astroquery.esasky.core] ...