Merge pull request #171 from pynbody/integration-testing

Integration testing

.github/workflows/integration-test.yaml

@@ -0,0 +1,72 @@
+name: Perform test database build and verification
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [3.9]
+        TANGOS_TESTING_DB_BACKEND: [sqlite]
+    runs-on: ${{ matrix.os }}
+    env:
+      C: gcc-10
+      CXX: g++-10
+    steps:
+    - name: Install Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - uses: actions/checkout@v2
+
+    - name: Update python pip/setuptools/wheel
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+
+    - name: Build and install tangos
+      run: |
+        pip install -e .
+
+    - name: Install latest pynbody
+      run: python -m pip install pynbody
+
+    - name: Cache test datasets
+      id: cache-test-datasets
+      uses: actions/cache@v2
+      with:
+        path: |
+          test_tutorial_build/tutorial_*
+          test_tutorial_build/reference_database.db
+        key: replace-later-with-md5 # need to work out a way to generate a key here at some point if test data changes
+
+    - name: Fetch test datasets
+      if: steps.cache-test-datasets.outputs.cache-hit != 'true'
+      working-directory: test_tutorial_build
+      run: |
+        wget -T 60 -nv ftp://zuserver2.star.ucl.ac.uk/app/tangos/mini_tutorial_test.tar.gz 
+        tar -xzvf mini_tutorial_test.tar.gz
+
+    - name: Build test database
+      working-directory: test_tutorial_build
+      run: bash build.sh
+
+    - uses: actions/upload-artifact@v2
+      with:
+        name: Tangos database
+        path: test_tutorial_build/data.db
+
+    - name: Verify database
+      working-directory: test_tutorial_build
+      run: tangos diff data.db reference_database.db --ignore-value-of gas_map gas_map_faceon gas_map_sideon uvi_image uvi_image_sideon uvi_image_faceon
+      # ignore-value-of above is a clunky fix for the use of 'approximate fast' images. Better solution would be good in the long term.
+

tangos/properties/pynbody/profile.py

@@ -28,7 +28,7 @@ def _get_profile(self, halo, maxrad):
 
         delta = self.plot_xdelta()
         nbins = int(maxrad / delta)
-        maxrad = delta * (nbins + 1)
+        maxrad = delta * nbins
 
         pro = pynbody.analysis.profile.Profile(halo, type='lin', ndim=3,
                                                min=0, max=maxrad, nbins=nbins)

tangos/scripts/__init__.py

@@ -45,7 +45,6 @@ def main(argv=None):
 
     from .. import core
     core.process_options(args)
-    core.init_db()
     args.func(args)
 
 

tangos/scripts/manager.py

@@ -414,7 +414,7 @@ def format_handler_name(cl):
 
 def diff(options):
     from ..testing import db_diff
-    differ = db_diff.TangosDbDiff(options.uri1, options.uri2)
+    differ = db_diff.TangosDbDiff(options.uri1, options.uri2, ignore_keys=options.ignore_value_of)
     if options.simulation:
         differ.compare_simulation(options.simulation)
     elif options.timestep:
@@ -423,11 +423,14 @@ def diff(options):
         differ.compare_object(options.object)
     else:
         differ.compare()
-    return differ.failed
+    result = differ.failed
 
-    diff = db_diff.diff(options.uri1, options.uri2)
-    if diff:
+    if result:
+        logger.info("Differences found. Exiting with status 1.")
         sys.exit(1)
+    else:
+        logger.info("No differences found.")
+
 
 def main():
     print("""
@@ -439,7 +442,6 @@ def main():
 
     args = parser.parse_args()
     core.process_options(args)
-    core.init_db()
     args.func(args)
 
 
@@ -533,6 +535,7 @@ def get_argument_parser_and_subparsers():
     subparse_diff.add_argument("--simulation", type=str, help="Only compare the specified simulation", default=None)
     subparse_diff.add_argument("--timestep", type=str, help="Only compare the specified timestep", default=None)
     subparse_diff.add_argument("--object", type=str, help="Only compare the specified object", default=None)
+    subparse_diff.add_argument("--ignore-value-of", nargs="*", type=str, help="Ignore the value of the specified properties", default=None)
     subparse_diff.set_defaults(func=diff)
 
     subparse_list_available_properties = subparse.add_parser("list-possible-properties",

tangos/testing/db_diff.py

@@ -3,11 +3,12 @@
 import six
 import numpy.testing as npt
 from sqlalchemy.orm import joinedload, object_session, undefer, Session
+import numpy as np
 
 class TangosDbDiff(object):
     """Class to compare two databases, used by the tangos diff command line tool"""
 
-    def __init__(self, uri1, uri2, max_objects=10):
+    def __init__(self, uri1, uri2, max_objects=10, ignore_keys=[]):
         if isinstance(uri1, Session):
             self.session1 = uri1
             uri1 = str(uri1.connection().engine)
@@ -28,11 +29,15 @@ def __init__(self, uri1, uri2, max_objects=10):
         self.test_objects = True
         self.max_objects = max_objects
         self.test_properties = True
+        self.ignore_keys = ignore_keys
         self.failed = False
 
         logger.info("Database 1 = %s",uri1)
         logger.info("Database 2 = %s",uri2)
         logger.info("For each timestep will check data in first %d objects of each type",max_objects)
+        if len(ignore_keys)>0:
+            logger.info("The value of properties with the names %s will be ignored.",", ".join(ignore_keys))
+            logger.info("Only the existence and type of these properties will be checked.")
 
     def fail(self, message, *args):
         logger.error(message, *args)
@@ -59,7 +64,7 @@ def compare_simulation(self, sim):
         ts1 = set([ts.extension for ts in sim1.timesteps])
         ts2 = set([ts.extension for ts in sim2.timesteps])
 
-        self._check_same_set(ts1, ts2, 'timesteps')
+        self._check_same_set(ts1, ts2, str(sim), 'timesteps')
 
         if self.test_timesteps:
             for ts in ts1.intersection(ts2):
@@ -74,7 +79,7 @@ def compare_timestep(self, ts):
 
         objects1 = dict([(o.path, o) for o in ts1.objects.filter(obj_filter).all()])
         objects2 = dict([(o.path, o) for o in ts2.objects.filter(obj_filter).all()])
-        self._check_same_set(objects1.keys(), objects2.keys(), 'objects')
+        self._check_same_set(objects1.keys(), objects2.keys(), str(ts), 'objects')
 
 
         if self.test_objects:
@@ -100,44 +105,59 @@ def _compare_objects(self, obj1, obj2):
         properties1 = dict([(prop.name.text, prop.data_raw) for prop in self._joined_properties_load(obj1)])
         properties2 = dict([(prop.name.text, prop.data_raw) for prop in self._joined_properties_load(obj2)])
 
-        self._check_dict_same(properties1, properties2)
+        self._check_dict_same(properties1, properties2, obj1.path)
 
         links1 = dict([(link.relation.text+"->"+link.halo_to.path, link.weight) for link in self._joined_links_load(obj1)])
         links2 = dict([(link.relation.text+"->"+link.halo_to.path, link.weight) for link in self._joined_links_load(obj2)])
 
-        self._check_dict_same(links1, links2, 'links')
+        self._check_dict_same(links1, links2, obj1.path, 'links')
 
-    def _check_dict_same(self, properties1, properties2, name_of_things='properties'):
+    def _check_dict_same(self, properties1, properties2, name_of_object, name_of_things='properties'):
         prop1_names = set(properties1.keys())
         prop2_names = set(properties2.keys())
-        self._check_same_set(prop1_names, prop2_names, name_of_things)
-        self._check_almost_equal(properties1, properties2)
+        self._check_same_set(prop1_names, prop2_names, name_of_object, name_of_things)
+        self._check_almost_equal(properties1, properties2, name_of_object)
 
-    def _check_same_set(self, objects1, objects2, name_of_things):
+    def _check_same_set(self, objects1, objects2, name_of_object, name_of_things):
         objects1 = set(objects1)
         objects2 = set(objects2)
-        self._check_setdiff_null("Database 1", objects1, objects2, name_of_things)
-        self._check_setdiff_null("Database 2",objects2, objects1, name_of_things)
+        self._check_setdiff_null("Database 1", objects1, objects2, name_of_object, name_of_things)
+        self._check_setdiff_null("Database 2",objects2, objects1, name_of_object, name_of_things)
 
-    def _check_setdiff_null(self, name_of_db_1, objects1, objects2,  name_of_things):
+    def _check_setdiff_null(self, name_of_db_1, objects1, objects2,  name_of_object, name_of_things):
         if len(objects1 - objects2) > 0:
-            self.fail("%s has %d additional %s", name_of_db_1, len(objects1 - objects2), name_of_things)
+            self.fail("When comparing %s, %s has %d additional %s", name_of_object, name_of_db_1,
+                      len(objects1 - objects2), name_of_things)
             self._print_objects(objects1 - objects2)
 
     def _print_objects(self, objects):
         for o in objects:
             logger.info("  %s", o)
 
-    def _check_almost_equal(self, dict1, dict2):
+    def _check_almost_equal(self, dict1, dict2, name_of_object):
         for d in dict1.keys():
             if d in dict2:
                 v1 = dict1[d]
                 v2 = dict2[d]
-                try:
-                    self._assert_equal(v1, v2)
-                except AssertionError as e:
-                    self.fail("Value mismatch for key %s",d)
-                    self.fail("%s",e)
+                if d in self.ignore_keys:
+                    self._check_properties_compatible(v1, v2, d, name_of_object)
+                else:
+                    self._check_properties_equal(v1, v2, d, name_of_object)
+
+    def _check_properties_equal(self, v1, v2, d, name_of_object):
+        try:
+            self._assert_equal(v1, v2)
+        except AssertionError as e:
+            self.fail("When comparing %s, value mismatch for key %s", name_of_object, d)
+            self.fail("%s", e)
+
+    def _check_properties_compatible(self, v1, v2, name_of_property, name_of_object):
+        if type(v1) != type(v2):
+            self.fail("When comparing %s, types of key %s differ", name_of_object, name_of_property)
+        else:
+            if isinstance(v1, np.ndarray):
+                if v1.shape != v2.shape:
+                    self.fail("When comparing %s, array shapes of key %s differ", name_of_object, name_of_property)
 
     def _assert_equal(self, v1, v2):
         if v1 is None or v2 is None:

test_tutorial_build/build.sh

@@ -73,8 +73,8 @@ echo "Starting process in 5 seconds..."
 
 sleep 5
 
-TANGOS_DB_CONNECTION=`pwd`/data.db
-TANGOS_SIMULATION_FOLDER=`pwd`
+export TANGOS_DB_CONNECTION=`pwd`/data.db
+export TANGOS_SIMULATION_FOLDER=`pwd`
 
 detect_mpi