Py3 dev (#366)

* modifications for python 3

* modifications for python 3

* changes for python 3

* Upated .travis.yml

* fixing unit tests

* fixing unit tests

* fixing up .travis and fake_travis

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-dist: trusty
+dist: xenial
 services:
   - docker
 
@@ -10,15 +10,16 @@ services:
 notifications:
   email: false
 
-python : 2.7
+python : 3.7
 
 env:
     global:
-      - FERMI_DIR=$HOME/ScienceTools/x86_64-unknown-linux-gnu-libc2.17
       - SLAC_ST_BUILD=false
       - PIP_DEPS='coverage pytest-cov'
+      - PYTHON_VERSION=3.7
       - INSTALL_CMD='python setup.py install'
       - CONDA_CHANNELS=conda-forge
+      - CONDA_DOWNLOAD=Miniconda3-latest-Linux-x86_64.sh
       - CONDA_DEPS='gammapy numpy astropy scipy matplotlib pyyaml astropy-healpix'
       - CONDA2_DEPS='subprocess32 pytest'
       - FERMI_CONDA_CHANNELS='-c conda-forge/label/cf201901 -c fermi -c fermi/label/beta'
@@ -29,51 +30,22 @@ matrix:
     include:
 
         # The main build:
-        # Python 2, lastest tag of fermitools, all dependencies
+        # Python 3, lastest tag of fermitools, all dependencies
         - os: linux
           env: NAME=main
-               PYTHON_VERSION=2.7
-               CONDA_DOWNLOAD=Miniconda2-latest-Linux-x86_64.sh
-               CONDA_DEPS='gammapy=0.10 healpy=1.13.0 numpy astropy scipy matplotlib pyyaml'
-               ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV -c conda-forge -c fermi -c fermi/label/beta fermitools"
+               ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV -c conda-forge -c fermi -c fermi/label/beta fermitools=2.0.0"
 
-        # The old build:
-        # Python 2, fixed tag of fermitools, all dependencies
+        # Python 3.7, no Fermi ST, all other dependencies
         - os: linux
-          env: NAME=old
-               PYTHON_VERSION=2.7
-               CONDA_DOWNLOAD=Miniconda2-latest-Linux-x86_64.sh
-               CONDA_DEPS='gammapy=0.10 healpy=1.13.0 numpy astropy scipy matplotlib pyyaml'
-               ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV -c conda-forge/label/cf201901 -c fermi -c conda-forge fermitools=1.2.23"
-
-        # Python 3.6, no Fermi ST, all other dependencies
-        - os: linux
-          python : 3.7
-          dist : xenial
-          env: NAME=py37_st-no_dep-yes
-               PYTHON_VERSION=3.7
+          env: NAME=no-deps
                ST_INSTALL='echo'
                CONDA2_DEPS='pytest'
-               CONDA_DOWNLOAD=Miniconda3-latest-Linux-x86_64.sh
-
-        # Python 2, no Fermi ST, all other dependencies
-        - os: linux
-          env: NAME=py2_st-no_dep-yes
-               PYTHON_VERSION=2.7
-               ST_INSTALL='echo'
-               CONDA_DEPS='gammapy=0.10 healpy=1.13.0 numpy astropy scipy matplotlib pyyaml'
-               CONDA_DOWNLOAD=Miniconda2-latest-Linux-x86_64.sh
 
-    allow_failures:
-        # The Sphinx docs build
         # Python 3, no Fermi ST, all other dependencies
         - os: linux
-          python : 3.7
           env: NAME=docs
-               PYTHON_VERSION=3.7
                ST_INSTALL='echo'
                PIP_DEPS='coverage pytest-cov coveralls'
-               CONDA_DOWNLOAD=Miniconda3-latest-Linux-x86_64.sh
                CONDA2_DEPS='pytest sphinx sphinx_rtd_theme'
 
 #before_install:

dev_install.sh

@@ -1,9 +1,9 @@
 
 
 # Set this to point at your conda installation
-export CONDA_PATH=$HOME/anaconda2
+export CONDA_PATH=$HOME/miniconda2
 # Set this to the name of the conda environment you want for fermipy development
-export FERMI_CONDA_ENV=fermipy-dev-test
+export FERMIPY_CONDA_ENV=fermipy-dev
 
 # Don't touch these
 export INSTALL_CMD="python setup.py develop"

fake_travis.sh

@@ -1,55 +1,36 @@
 
 export SLAC_ST_BUILD=false
-export PIP_DEPS='coverage pytest-cov coveralls'
+export PIP_DEPS='coverage pytest-cov'
+export PYTHON_VERSION=3.7
 export INSTALL_CMD='python setup.py install'
+export CONDA_PATH="$HOME/miniconda3"
 export CONDA_CHANNELS="conda-forge"
-export FERMI_CONDA_CHANNELS="-c conda-forge/label/cf201901 -c fermi"
+export CONDA_DOWNLOAD="Miniconda3-latest-Linux-x86_64.sh"
 export CONDA_DEPS='gammapy numpy astropy scipy matplotlib pyyaml astropy-healpix'
 export CONDA2_DEPS='subprocess32 pytest'
+export FERMI_CONDA_CHANNELS="-c conda-forge/label/cf201901 -c fermi -c fermi/label/beta"
 export DOCKER_INSTALL=''
 
-NAME='py37'
+NAME='main'
 export FERMIPY_CONDA_ENV="fermipy-test-$NAME"
 
 
 case $NAME in
     main)
-	export PYTHON_VERSION="2.7"
-	export CONDA_PATH="/Users/echarles/miniconda2"
-	export CONDA_DEPS='gammapy=0.10 healpy=1.13.0 numpy astropy scipy matplotlib pyyaml'	
-	export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV -c conda-forge -c fermi -c fermi/label/beta fermitools"
+	export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV -c conda-forge -c fermi -c fermi/label/beta fermitools=2.0.0"
 	;;
-    old)
-	export PYTHON_VERSION="2.7"
-	export CONDA_PATH="/Users/echarles/miniconda2"
-	export CONDA_DEPS='gammapy=0.10 healpy=1.13.0 numpy astropy scipy matplotlib pyyaml'	
-	export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV -c conda-forge/label/cf201901 -c fermi -c conda-forge fermitools=1.2.23"
+    dev)
+	export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV -c conda-forge -c fermi -c fermi/label/dev fermitools=2.0.2"
 	;;
-    py37)
-	export PYTHON_VERSION="3.7"
-	export CONDA_PATH="/Users/echarles/miniconda3"
-        export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV -c conda-forge -c fermi -c fermi/label/beta fermitools"
+    no-deps)
+        export ST_INSTALL="echo"
         export CONDA2_DEPS='pytest'
 	;;
     docs)
-	export PYTHON_VERSION="3.7"
-	export CONDA_PATH="/Users/echarles/miniconda3"
-        export ST_INSTALL=''
+        export ST_INSTALL='echo'
 	export PIP_DEPS='coverage pytest-cov coveralls'
 	export CONDA2_DEPS='pytest sphinx sphinx_rtd_theme'
 	;;
-    py37_st-no_dep)
-	export PYTHON_VERSION="3.7"
-	export CONDA_PATH="/Users/echarles/miniconda3"
-        export ST_INSTALL=''
-        export CONDA2_DEPS='pytest'	
-	;;
-    py2_st-no_dep)
-	export PYTHON_VERSION="2.7"
-	export CONDA_PATH="/Users/echarles/miniconda2"
-	export CONDA_DEPS='gammapy=0.10 healpy=1.13.0 numpy astropy scipy matplotlib pyyaml'	
-        export ST_INSTALL=''
-	;;
     none)
 	exit
         ;;

fermipy/extension.py

@@ -92,7 +92,12 @@ def extension(self, name, **kwargs):
             self._make_extension_fits(ext, outfile + '.fits')
 
         if config['write_npy']:
-            np.save(outfile + '.npy', dict(ext))
+            o_copy = dict(ext)
+            self.logger.warning('Saving maps in .npy files is disabled b/c of incompatibilities in python3, remove the maps from the %s.npy' % outfile)
+            print(o_copy)
+            for xrm in ['tsmap']:
+                o_copy.pop(xrm)
+            np.save(outfile + '.npy', o_copy)
 
         self.logger.info('Execution time: %.2f s', timer.elapsed_time)
         return ext
@@ -368,8 +373,12 @@ def _make_extension_fits(self, ext, filename, **kwargs):
             hdus = [fits.PrimaryHDU()]
 
         hdus += [hdu_data] + hdu_images
-        hdus[0].header['CONFIG'] = json.dumps(utils.tolist(ext['config']))
-        hdus[1].header['CONFIG'] = json.dumps(utils.tolist(ext['config']))
+        ll = utils.tolist(ext['config'])
+        print(ext['config'])
+        print(ll)
+        jstring = json.dumps(ll)
+        hdus[0].header['CONFIG'] = jstring
+        hdus[1].header['CONFIG'] = jstring
         fits_utils.write_hdus(hdus, filename,
                               keywords={'SRCNAME': ext['name']})
 

fermipy/gtanalysis.py

@@ -412,18 +412,21 @@ def __init__(self, config, roi=None, **kwargs):
 
         axes = [MapAxis.from_edges(self.energies, interp='log',
                                    name='energy', unit='MeV')]
+
+        frame_map = dict(CEL='icrs', cel='icrs', GAL='galactic', gal='galactic')
+
         if self.projtype == 'HPX':
             is_nested = self.config['binning']['hpx_ordering_scheme'] == "NESTED"
             self._geom = HpxGeom.create(2**self.config['binning']['hpx_order'],
                                         nest=is_nested,
-                                        coordsys=self.config['binning']['coordsys'],
+                                        frame=frame_map[self.config['binning']['coordsys']],
                                         width=self.config['binning']['roiwidth'],
                                         skydir=self.roi.skydir,
                                         axes=axes)
         else:
 
             self._geom = WcsGeom.create(npix=self.npix, binsz=self._binsz,
-                                        coordsys=self.config['binning']['coordsys'],
+                                        frame=frame_map[self.config['binning']['coordsys']],
                                         proj=self.config['binning']['proj'],
                                         skydir=self.roi.skydir,
                                         axes=axes)
@@ -603,8 +606,13 @@ def _update_roi(self):
         rm['npred'] = 0
         rm['npred_wt'] = 0
         for i, c in enumerate(self.components):
+
             rm['components'][i]['loglike'] = -c.like()
-            rm['components'][i]['model_counts'].fill(0)
+            try:
+                rm['components'][i]['model_counts'].fill(0)
+            except KeyError:
+                raise ValueError("%s" % rm['components'][i].keys())
+
             try:
                 rm['components'][i]['model_counts_wt'].fill(0)
             except:
@@ -939,12 +947,13 @@ def _create_component(self, cfg, **kwargs):
 
         cfg['fileio']['workdir'] = self.config['fileio']['workdir']
 
+        copy_cfg = {}
         for k in cfg.keys():
-            if not k in GTBinnedAnalysis.defaults:
-                cfg.pop(k)
+            if k in GTBinnedAnalysis.defaults:
+                copy_cfg[k] = cfg[k]
 
         comp = GTBinnedAnalysis(
-            cfg, self.roi, logging=self.config['logging'], **kwargs)
+            copy_cfg, self.roi, logging=self.config['logging'], **kwargs)
 
         return comp
 
@@ -1077,10 +1086,12 @@ def setup(self, init_sources=True, overwrite=False, **kwargs):
 
         # Determine tmin, tmax
         for i, c in enumerate(self._components):
-            self._tmin = (c.tmin if self._tmin is None
-                          else min(self._tmin, c.tmin))
-            self._tmax = (c.tmax if self._tmax is None
-                          else min(self._tmax, c.tmax))
+            if c.tmin is not None:
+                self._tmin = (c.tmin if self._tmin is None
+                            else min(self._tmin, c.tmin))
+            if c.tmax is not None:
+                self._tmax = (c.tmax if self._tmax is None
+                                else min(self._tmax, c.tmax))
 
         if init_sources:
 
@@ -1461,6 +1472,9 @@ def delete_source(self, name, save_template=True, delete_source_map=False,
         if self.like is not None:
             self.like.model = self.like.components[0].model
             self._update_roi()
+
+        if self.roi.has_source(name):
+            raise ValueError("delete source failed for %s" % name)
         return src
 
     def delete_sources(self, cuts=None, distance=None,
@@ -3615,7 +3629,7 @@ def load_roi(self, infile, reload_sources=False, params=None, mask=None):
                    'ref_dfde_emax': 'ref_dnde_e_max',
                    }
 
-        self._roi_data = utils.update_keys(roi_data['roi'], key_map)
+        self._roi_data = roi_data['roi']
 
         if 'erange' in self._roi_data:
             self._roi_data['loge_bounds'] = self._roi_data.pop('erange')
@@ -4463,19 +4477,26 @@ def __init__(self, config, roi, **kwargs):
         self._coordsys = self.config['binning']['coordsys']
         self._tmin = self.config['selection']['tmin']
         self._tmax = self.config['selection']['tmax']
+
+        if self._tmin is None or self._tmax is None:
+            raise ValueError("%s %s" % (self._tmin, self._tmax))
+
         axes = [MapAxis.from_edges(self.energies, interp='log',
                                    name='energy', unit='MeV')]
+
+        frame_map = dict(CEL='icrs', cel='icrs', GAL='galactic', gal='galactic')
+
         if self.projtype == 'HPX':
             is_nested = self.config['binning']['hpx_ordering_scheme'] == "NESTED"
             self._geom = HpxGeom.create(2**self.config['binning']['hpx_order'],
                                         nest=is_nested,
-                                        coordsys=self.config['binning']['coordsys'],
+                                        frame=frame_map[self.config['binning']['coordsys']],
                                         width=self.config['binning']['roiwidth'],
                                         skydir=self.roi.skydir,
                                         axes=axes)
         elif self.projtype == "WCS":
             self._geom = WcsGeom.create(npix=self.npix, binsz=self.binsz,
-                                        coordsys=self.config['binning']['coordsys'],
+                                        frame=frame_map[self.config['binning']['coordsys']],
                                         proj=self.config['binning']['proj'],
                                         skydir=self.roi.skydir,
                                         axes=axes)

fermipy/irfs.py

@@ -800,7 +800,7 @@ def create_wtd_psf(skydir, ltc, event_class, event_types, dtheta,
     """
     #npts = int(np.ceil(32. / bins_per_dec(egy_bins)))
     egy_bins = np.exp(utils.split_bin_edges(np.log(egy_bins), npts))
-    etrue_bins = 10**np.linspace(1.0, 6.5, nbin * 5.5 + 1)
+    etrue_bins = 10**np.linspace(1.0, 6.5, np.int(np.ceil(nbin * 5.5 + 1)))
     etrue = 10**utils.edge_to_center(np.log10(etrue_bins))
 
     psf = create_avg_psf(skydir, ltc, event_class, event_types, dtheta,
@@ -832,7 +832,7 @@ def calc_drm(skydir, ltc, event_class, event_types,
     npts = int(np.ceil(128. / bins_per_dec(egy_bins)))
     egy_bins = np.exp(utils.split_bin_edges(np.log(egy_bins), npts))
 
-    etrue_bins = 10**np.linspace(1.0, 6.5, nbin * 5.5 + 1)
+    etrue_bins = 10**np.linspace(1.0, 6.5, np.int(np.ceil(nbin * 5.5 + 1)))
     egy = 10**utils.edge_to_center(np.log10(egy_bins))
     egy_width = utils.edge_to_width(egy_bins)
     etrue = 10**utils.edge_to_center(np.log10(etrue_bins))
@@ -901,7 +901,7 @@ def calc_counts_edisp(skydir, ltc, event_class, event_types,
 
     # Split energy bins
     egy_bins = np.exp(utils.split_bin_edges(np.log(egy_bins), npts))
-    etrue_bins = 10**np.linspace(1.0, 6.5, nbin * 5.5 + 1)
+    etrue_bins = 10**np.linspace(1.0, 6.5, np.int(np.ceil(nbin * 5.5 + 1)))
     drm = calc_drm(skydir, ltc, event_class, event_types,
                    egy_bins, cth_bins, nbin=nbin)
     cnts_etrue = calc_counts(skydir, ltc, event_class, event_types,

fermipy/jobs/tests/test_gtlink.py

@@ -39,7 +39,7 @@ class Gtlink_scrmaps(Gtlink):
 
     gtlink = Gtlink_scrmaps()
     formatted_command = gtlink.formatted_command()
-    assert formatted_command == 'gtsrcmaps irfs=CALDB expcube=none cmap=none srcmdl=none outfile=none bexpmap=none'
+    assert formatted_command == 'gtsrcmaps irfs=CALDB expcube=none bexpmap=none cmap=none srcmdl=none outfile=none'
 
 
 if __name__ == '__main__':