Merge pull request #3118 from andamian/CADC-13741

Added support for temporary table upload in `alma`

CHANGES.rst

@@ -22,6 +22,8 @@ alma
 
 - Added support for frequency_resolution in KHz [#3035]
 
+- Added support for temporary upload tables in query_tap [#3118]
+
 - Changed the way galactic ranges are used in queries [#3105]
 
 ehst

astroquery/alma/core.py

@@ -138,7 +138,7 @@
     'Project': {
         'Project code': ['project_code', 'proposal_id', _gen_str_sql],
         'Project title': ['project_title', 'obs_title', _gen_str_sql],
-        'PI name': ['pi_name', 'obs_creator_name', _gen_str_sql],
+        'PI name': ['pi_name', 'pi_name', _gen_str_sql],
         'Proposal authors': ['proposal_authors', 'proposal_authors', _gen_str_sql],
         'Project abstract': ['project_abstract', 'proposal_abstract', _gen_str_sql],
         'Publication count': ['publication_count', 'NA', _gen_str_sql],
@@ -678,19 +678,33 @@ def query_sia(self, *, pos=None, band=None, time=None, pol=None,
 
     query_sia.__doc__ = query_sia.__doc__.replace('_SIA2_PARAMETERS', SIA2_PARAMETERS_DESC)
 
-    def query_tap(self, query, maxrec=None):
+    def query_tap(self, query, *, maxrec=None, uploads=None):
         """
         Send query to the ALMA TAP. Results in pyvo.dal.TapResult format.
         result.table in Astropy table format
 
         Parameters
         ----------
+        query : str
+            ADQL query to execute
         maxrec : int
             maximum number of records to return
+        uploads : dict
+            a mapping from temporary table names to objects containing a votable. These
+            temporary tables can be referred to in queries. The keys in the dictionary are
+            the names of temporary tables which need to be prefixed with the TAP_UPLOAD
+            schema in the actual query. The values are either astropy.table.Table instances
+            or file names or file like handles such as io.StringIO to table definition in
+            IVOA VOTable format.
+
+    Examples
+    --------
+    >>> uploads = {'tmptable': '/tmp/tmptable_def.xml'}
+    >>> rslt = query_tap(self, query, maxrec=None, uploads=uploads)
 
         """
         log.debug('TAP query: {}'.format(query))
-        return self.tap.search(query, language='ADQL', maxrec=maxrec)
+        return self.tap.search(query, language='ADQL', maxrec=maxrec, uploads=uploads)
 
     def help_tap(self):
         print('Table to query is "voa.ObsCore".')

astroquery/alma/tests/test_alma.py

@@ -269,7 +269,7 @@ def test_query():
         "select * from ivoa.obscore WHERE "
         "(INTERSECTS(CIRCLE('ICRS',1.0,2.0,1.0), s_region) = 1) "
         "AND science_observation='T' AND data_rights='Public'",
-        language='ADQL', maxrec=None)
+        language='ADQL', maxrec=None, uploads=None)
 
     # one row result
     tap_mock = Mock()
@@ -291,7 +291,7 @@ def test_query():
         "(INTERSECTS(CIRCLE('ICRS',1.0,2.0,0.16666666666666666), s_region) = 1) "
         "AND band_list LIKE '%3%' AND science_observation='T' AND "
         "data_rights='Proprietary'",
-        language='ADQL', maxrec=None)
+        language='ADQL', maxrec=None, uploads=None)
 
     # repeat for legacy columns
     mock_result = Mock()
@@ -313,7 +313,7 @@ def test_query():
         "(INTERSECTS(CIRCLE('ICRS',1.0,2.0,0.16666666666666666), s_region) = 1) "
         "AND band_list LIKE '%3%' AND science_observation='T' AND "
         "data_rights='Proprietary'",
-        language='ADQL', maxrec=None)
+        language='ADQL', maxrec=None, uploads=None)
     row_legacy = result_legacy[0]
     row = result[0]
     for item in _OBSCORE_TO_ALMARESULT.items():
@@ -347,7 +347,7 @@ def test_query():
         "(band_list LIKE '%1%' OR band_list LIKE '%3%') AND "
         "t_min=55197.0 AND pol_states='/XX/YY/' AND s_fov=0.012313 AND "
         "t_exptime=25 AND science_observation='F'",
-        language='ADQL', maxrec=None
+        language='ADQL', maxrec=None, uploads=None
     )
 
     tap_mock.reset()
@@ -361,7 +361,7 @@ def test_query():
         "AND spectral_resolution=2000000 "
         "AND (INTERSECTS(CIRCLE('ICRS',1.0,2.0,1.0), "
         "s_region) = 1) AND science_observation='T' AND data_rights='Public'",
-        language='ADQL', maxrec=None)
+        language='ADQL', maxrec=None, uploads=None)
 
 
 @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyUserWarning")
@@ -499,9 +499,14 @@ def test_tap():
     alma._tap = tap_mock
     result = alma.query_tap('select * from ivoa.ObsCore')
     assert len(result.table) == 0
-
     tap_mock.search.assert_called_once_with('select * from ivoa.ObsCore',
-                                            language='ADQL', maxrec=None)
+                                            language='ADQL', maxrec=None, uploads=None)
+
+    tap_mock.search.reset_mock()
+    result = alma.query_tap('select * from ivoa.ObsCore', maxrec=10, uploads={'tmptable': 'votable_file.xml'})
+    assert len(result.table) == 0
+    tap_mock.search.assert_called_once_with(
+        'select * from ivoa.ObsCore', language='ADQL', maxrec=10, uploads={'tmptable': 'votable_file.xml'})
 
 
 @pytest.mark.parametrize('data_archive_url',

astroquery/alma/tests/test_alma_remote.py

@@ -1,6 +1,7 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 from datetime import datetime, timezone
 import os
+from io import StringIO
 from pathlib import Path
 from urllib.parse import urlparse
 import re
@@ -23,7 +24,6 @@
 except ImportError:
     HAS_REGIONS = False
 
-# ALMA tests involving staging take too long, leading to travis timeouts
 # TODO: make this a configuration item
 SKIP_SLOW = True
 
@@ -91,14 +91,15 @@ def test_SgrAstar(self, tmp_path, alma):
 
         assert '2013.1.00857.S' in result_s['Project code']
 
+    @pytest.mark.skipif("SKIP_SLOW")
     def test_freq(self, alma):
         payload = {'frequency': '85..86'}
         result = alma.query(payload)
         assert len(result) > 0
         for row in result:
             # returned em_min and em_max are in m
             assert row['frequency'] >= 85
-            assert row['frequency'] <= 100
+            assert row['frequency'] <= 86
             assert '3' in row['band_list']
 
     def test_bands(self, alma):
@@ -216,10 +217,7 @@ def test_data_info(self, tmp_path, alma):
         trimmed_access_url_list = [e for e in data_info_tar['access_url'].data if len(e) > 0]
         trimmed_access_urls = (trimmed_access_url_list,)
         mock_calls = download_files_mock.mock_calls[0][1]
-        print(f"\n\nComparing {mock_calls} to {trimmed_access_urls}\n\n")
-        # comparison = download_files_mock.mock_calls[0][1] == data_info_tar['access_url']
         assert mock_calls == trimmed_access_urls
-        # assert comparison.all()
 
     def test_download_data(self, tmp_path, alma):
         # test only fits files from a program
@@ -239,9 +237,8 @@ def test_download_data(self, tmp_path, alma):
         alma._download_file.call_count == len(results)
         assert len(results) == len(urls)
 
+    @pytest.mark.skipif("SKIP_SLOW")
     def test_download_and_extract(self, tmp_path, alma):
-        # TODO: slowish, runs for ~90s
-
         alma.cache_location = tmp_path
         alma._cycle0_tarfile_content_table = {'ID': ''}
 
@@ -345,16 +342,18 @@ def test_misc(self, alma):
 
         result = alma.query_object('M83', public=True, science=True)
         assert len(result) > 0
-        result = alma.query(payload={'pi_name': '*Bally*'}, public=False,
-                            maxrec=10)
+        with pytest.warns(expected_warning=DALOverflowWarning,
+                          match="Partial result set. Potential causes MAXREC, async storage space, etc."):
+            result = alma.query(payload={'pi_name': 'Bally*'}, public=True,
+                                maxrec=10)
         assert result
         # Add overwrite=True in case the test previously died unexpectedly
         # and left the temp file.
         result.write('/tmp/alma-onerow.txt', format='ascii', overwrite=True)
         for row in result:
-            assert 'Bally' in row['obs_creator_name']
+            assert 'Bally' in row['pi_name']
         result = alma.query(payload=dict(project_code='2016.1.00165.S'),
-                            public=False)
+                            public=True)
         assert result
         for row in result:
             assert '2016.1.00165.S' == row['proposal_id']
@@ -398,8 +397,9 @@ def test_misc(self, alma):
         assert result
         for row in result:
             assert '6' == row['band_list']
-            assert 'ginsburg' in row['obs_creator_name'].lower()
+            assert 'ginsburg' in row['pi_name'].lower()
 
+    @pytest.mark.skip("Not sure what this is supposed to do")
     def test_user(self, alma):
         # miscellaneous set of tests from current users
         rslt = alma.query({'band_list': [6], 'project_code': '2012.1.*'},
@@ -561,6 +561,36 @@ def test_big_download_regression(alma):
     alma.download_files([files['access_url'][3]])
 
 
+@pytest.mark.remote_data
+def test_tap_upload():
+    tmp_table = StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+    <VOTABLE xmlns="http://www.ivoa.net/xml/VOTable/v1.3"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.3">
+      <RESOURCE>
+        <TABLE>
+          <FIELD name="prop_id" datatype="char" arraysize="*">
+            <DESCRIPTION>external URI for the physical artifact</DESCRIPTION>
+          </FIELD>
+          <DATA>
+            <TABLEDATA>
+              <TR>
+                <TD>2013.1.01365.S</TD>
+              </TR>
+            </TABLEDATA>
+          </DATA>
+        </TABLE>
+      </RESOURCE>
+    </VOTABLE>''')
+
+    alma = Alma()
+    res = alma.query_tap(
+        'select top 3 proposal_id from ivoa.ObsCore oc join TAP_UPLOAD.proj_codes pc on oc.proposal_id=pc.prop_id',
+        uploads={'proj_codes': tmp_table})
+    assert len(res) == 3
+    for row in res:
+        assert row['proposal_id'] == '2013.1.01365.S'
+
+
 @pytest.mark.remote_data
 def test_download_html_file(alma, tmp_path):
     alma.cache_location = tmp_path

docs/alma/alma.rst

@@ -178,7 +178,7 @@ or if you wanted all projects by a given PI:
 
     >>> Alma.query(payload=dict(pi_name='Ginsburg, Adam'))
 
-The ''query_sia'' method offers another way to query ALMA using the IVOA SIA
+The ``query_sia`` method offers another way to query ALMA using the IVOA SIA
 subset of keywords returning results in 'ObsCore' format.  For example,
 to query for all images that have ``'XX'`` polarization (note that this query is too large
 to run, it is just shown as an example):
@@ -187,9 +187,9 @@ to run, it is just shown as an example):
 
     >>> Alma.query_sia(pol='XX')  # doctest: +SKIP
 
-Finally, the ''query_tap'' method is the most general way of querying the ALMA
+Finally, the ``query_tap`` method is the most general way of querying the ALMA
 metadata. This method is used to send queries to the service using the
-'ObsCore' columns as constraints. The returned result is also in 'ObsCore'
+``ObsCore`` columns as constraints. The returned result is also in ``ObsCore``
 format.
 
 .. doctest-remote-data::
@@ -210,8 +210,37 @@ One can also query by keyword, spatial resolution, etc:
     ...                "in ('Disks around high-mass stars', 'Asymptotic Giant Branch (AGB) stars') "
     ...                "AND science_observation='T'")  # doctest: +IGNORE_OUTPUT
 
+``query_tap`` also supports uploading temporary tables that can be used to join to in queries.
+These temporary tables can be defined as ''astropy.table.Table'' instances or references to file names or
+file like handles (`~io.StringIO` instances for example) of table definitions in IVOA VOTable format.
+Below is a very simple example of using `~astropy.table.Table` temporary table with the ``proj_codes`` name.
+Note that the table name must always be prefixed with the ``TAP_UPLOAD`` schema when referenced in queries.
 
-Use the ``help_tap`` method to learn about the ALMA 'ObsCore' keywords and
+.. doctest-remote-data::
+
+    >>> from astropy.table import Table
+    >>> tmp_table = Table([['2013.1.01365.S', '2013.A.00014.S']], names=['prop_id'], dtype=['S'])
+    >>> Alma.query_tap('select distinct target_name from ivoa.ObsCore oc join TAP_UPLOAD.proj_codes pc on oc.proposal_id=pc.prop_id order by target_name',
+    ...                uploads={'proj_codes': tmp_table})
+    <DALResultsTable length=13>
+    target_name
+       str256
+    ------------
+           Ceres
+      J0042-4030
+      J0334-4008
+       J1733-130
+      J1751+0939
+       J1751+096
+      J1851+0035
+      J1924-2914
+         Neptune
+    SGP-UR-54092
+           Titan
+          Uranus
+         W43-MM1
+
+Use the ``help_tap`` method to learn about the ALMA ``ObsCore`` keywords and
 their types.
 
 .. doctest-remote-data::