diff --git a/CHANGELOG b/CHANGELOG index 4d83c4a..7d224cc 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,9 +1,17 @@ +0.6.5 +- Sessions are now autosaved every 60s (#97) +- Bugfixes: + - Loading of sessions with hierarchy children failed (#99) + - Reduced a memory leak for new analyses that resulted in frequent + memory errors (should be less frequent now) 0.6.4 - Bugfixes: - Batch analysis: Individual measurement parameters not preserved (#96) - Legend plot: >9 measurements cannot be displayed in legend at the same time. Introduced new config key "Legend Autoscaled" (#91) - New filter hierarchy allows to investigate subpopulations (#87, #63) +- dclab + - Add InertiaRatio and InertiaRatioRaw to tdms files (typo) 0.6.3 - Bugfixes: - Regression: Excluded events plotted on top of filtered events (#86) diff --git a/appveyor.yml b/appveyor.yml index 83411af..25c87aa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,7 +6,8 @@ # Do not use shallow_clone, because coveralls needs a git repository #shallow_clone: true -clone_depth: 10 +# Use large clone depth so that a tag is included for versioning +clone_depth: 256 # Only build master branch @@ -86,6 +87,7 @@ install: # https://ci.appveyor.com/project/paulmueller/shapeout/build/1.0.152#L504 - "pip freeze" # PIP installs + - "pip install appdirs" - "pip install nptdms" # install pyper (R) - "pip install pyper" diff --git a/requirements.txt b/requirements.txt index eef18a4..bb79707 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +appdirs chaco>=4.1.0 cffi>=0.8.2 dclab>=0.1.0 diff --git a/setup.py b/setup.py index 592b862..9de7308 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ 'GUI': ["wxPython", "chaco", "opencv"], # kiwisolver? }, - install_requires=["dclab", "NumPy>=1.7.0", "SciPy>=0.10.0", + install_requires=["appdirs", "dclab", "NumPy>=1.7.0", "SciPy>=0.10.0", "pyper"], setup_requires=['pytest-runner'], tests_require=["pytest", "urllib3"], diff --git a/shapeout/ShapeOut.py b/shapeout/ShapeOut.py index 5865ae5..656daf2 100644 --- a/shapeout/ShapeOut.py +++ b/shapeout/ShapeOut.py @@ -32,6 +32,7 @@ def main(): # close the splash screen splash.terminate() # launch application + app.frame.InitRun() app.MainLoop() diff --git a/shapeout/__main__.py b/shapeout/__main__.py index 7fa6e5f..106c362 100644 --- a/shapeout/__main__.py +++ b/shapeout/__main__.py @@ -58,19 +58,20 @@ def prepare_app(): warnings.warn(_("Could not determine ShapeOut version.")) version = None + app.frame = frontend.Frame(version) + + return app + +if __name__ == "__main__": # get session file session_file = None for arg in sys.argv: if arg.endswith(".zmso"): - print("\nLoading Session "+arg) + print("\nUsing Session "+arg) session_file=arg else: print("Ignoring command line parameter: "+arg) - app.frame = frontend.Frame(version, session_file=session_file) - - return app - -if __name__ == "__main__": app = prepare_app() + app.frame.InitRun(session_file=session_file) app.MainLoop() diff --git a/shapeout/analysis.py b/shapeout/analysis.py index 8f334a2..dce6e1b 100644 --- a/shapeout/analysis.py +++ b/shapeout/analysis.py @@ -11,6 +11,7 @@ import copy import numpy as np import os +import sys import warnings # dclab imports @@ -53,6 +54,41 @@ def __init__(self, data, search_path="./"): " .tdms files: {}".format(data)) + def _clear(self): + """Remove all attributes from this instance, making it unusable + + It is difficult to control how the chaco plots refer to a measurement + object. + + """ + import gc + for _i in range(len(self.measurements)): + mm = self.measurements.pop(0) + # Deleting all the data in measurements! + attrs = dclab.definitions.rdv + attrs += ["_filter_"+a for a in attrs] + attrs += ["_filter", "_plot_filter", "_Downsampled_Scatter"] + for a in attrs: + if hasattr(mm, a): + b = getattr(mm, a) + del b + # Also delete fluorescence curves + for tr in list(mm.traces.keys()): + t = mm.traces.pop(tr) + del t + del mm.traces + refs = gc.get_referrers(mm) + for r in refs: + if hasattr(r, "delplot"): + r.delplot() + import IPython + IPython.embed() + del r + del mm + dclab.cached.Cache.clear_cache() + gc.collect() + + def _ImportDumped(self, indexname, search_path="./"): """ Loads data from index file as saved using `self.DumpData`. @@ -96,8 +132,12 @@ def _ImportDumped(self, indexname, search_path="./"): ids = [mm.identifier for mm in measmts if mm is not None] mms = [mm for mm in measmts if mm is not None] if idhp in ids: + # parent exists hparent = mms[ids.index(idhp)] mm = RTDC_DataSet(hparent=hparent) + else: + # parent doesn't exist - try again in next loop + continue else: tloc = session_get_tdms_file(data, search_path) mm = RTDC_DataSet(tloc) @@ -195,9 +235,8 @@ def DumpData(self, directory, fullout=False, rel_path="./"): for i in range(len(out)): out[i] += "\r\n" - index = codecs.open(indexname, "w", "utf-8") - index.writelines(out) - index.close() + with codecs.open(indexname, "w", "utf-8") as index: + index.writelines(out) # Dump polygons if len(PolygonFilter.instances) > 0: diff --git a/shapeout/gui/autosave.py b/shapeout/gui/autosave.py new file mode 100644 index 0000000..a3d9f13 --- /dev/null +++ b/shapeout/gui/autosave.py @@ -0,0 +1,77 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" ShapeOut - autosaving of sessions +""" +from __future__ import division, print_function + +import appdirs +import os +import time +import wx +import wx.lib.delayedresult as delayedresult + +from . import session + +cache_dir = appdirs.user_cache_dir(appname="ShapeOut") +autosave_file = os.path.join(cache_dir, "autosave.zmso") + + +def mkdir_p(adir): + """Recursively create a directory""" + adir = os.path.abspath(adir) + while not os.path.exists(adir): + try: + os.mkdir(adir) + except: + mkdir_p(os.path.dirname(adir)) + + +def _autosave_consumer(delayedresult, parent): + parent.StatusBar.SetStatusText("Autosaving...") + tempname = autosave_file+".tmp" + mkdir_p(cache_dir) + try: + session.save_session(tempname, parent.analysis) + except: + parent.StatusBar.SetStatusText("Autosaving failed!") + if os.path.exists(tempname): + os.remove(tempname) + else: + if os.path.exists(autosave_file): + os.remove(autosave_file) + os.rename(tempname, autosave_file) + parent.StatusBar.SetStatusText("") + autosave_run(parent) + + +def _autosave_worker(parent, interval): + """Runs in the background and performs autosaving""" + time.sleep(interval) + + +def autosave_run(parent, interval=60): + """Runs in the background and performs autosaving""" + delayedresult.startWorker(_autosave_consumer, + _autosave_worker, + wargs=(parent, interval,), + cargs=(parent,)) + + +def check_recover(parent): + """Check for a recovery file and ask if user wants to restore + + Returns True if a session was restored. False otherwise. + """ + if os.path.exists(autosave_file): + message="Autosaved session found. Restore?" + dlg = wx.MessageDialog(parent, + caption=_("Missing tdms files for session"), + message=message, + style=wx.YES|wx.NO, + ) + mod = dlg.ShowModal() + dlg.Destroy() + if mod == wx.ID_YES: + session.open_session(autosave_file, parent) + return True + return False diff --git a/shapeout/gui/frontend.py b/shapeout/gui/frontend.py index 9ddf49b..d5450d2 100644 --- a/shapeout/gui/frontend.py +++ b/shapeout/gui/frontend.py @@ -11,12 +11,9 @@ import numpy as np import os import platform -import shutil import sys -import tempfile import traceback import wx -import zipfile import dclab @@ -27,6 +24,7 @@ import gaugeframe from .. import analysis from .. import tlabwrap +from . import autosave from . import update from . import plot_main from . import misc @@ -34,6 +32,8 @@ from . import export from . import batch from . import plot_export +from . import session + ######################################################################## class ExceptionDialog(wx.MessageDialog): @@ -47,7 +47,7 @@ def __init__(self, msg): ######################################################################## class Frame(gaugeframe.GaugeFrame): """""" - def __init__(self, version, session_file=None): + def __init__(self, version): """Constructor""" self.config = ConfigurationFile(findfile("shapeout.cfg")) self.version = version @@ -125,12 +125,6 @@ def __init__(self, version, session_file=None): self.spright.SetMinimumPaneSize(100) self.sptop.SetMinimumPaneSize(100) - if session_file is not None: - self.OnMenuLoad(session_file=session_file) - - update.Update(self) - - # Set window icon try: self.MainIcon = misc.getMainIcon() @@ -138,6 +132,27 @@ def __init__(self, version, session_file=None): except: self.MainIcon = None + + def InitRun(self, session_file=None): + """Performs the first tasks after the publication starts + + - start autosaving + - check for updates + """ + # Check if we have an autosaved session that we did not delete + recover = autosave.check_recover(self) + + # Load session file if provided + if session_file is not None and not recover: + self.OnMenuLoad(session_file=session_file) + + # Search for updates + update.Update(self) + + # Start autosaving + autosave.autosave_run(self) + + def InitUI(self): """Menus, Toolbar, Statusbar""" @@ -272,6 +287,7 @@ def NewAnalysis(self, data, search_path="./"): anal = analysis.Analysis(data, search_path=search_path) # Get Plotting and Filtering parameters from previous analysis if hasattr(self, "analysis"): + # Get Plotting and Filtering parameters from previous analysis fpar = self.analysis.GetParameters("Filtering") ppar = self.analysis.GetParameters("Plotting") newcfg = {"Filtering" : fpar, @@ -286,6 +302,7 @@ def NewAnalysis(self, data, search_path="./"): # - only works if len(colors) matches number of measurements colors = self.analysis.GetContourColors() anal.SetContourColors(colors) + self.analysis._clear() self.analysis = anal self.PanelTop.NewAnalysis(anal) self.PlotArea.Plot(anal) @@ -454,91 +471,8 @@ def OnMenuLoad(self, e=None, session_file=None): dirname = os.path.dirname(fname) self.config.SetWorkingDirectory(dirname) - Arc = zipfile.ZipFile(fname, mode='r') - tempdir = tempfile.mkdtemp() - Arc.extractall(tempdir) - Arc.close() - - indexfile = os.path.join(tempdir, "index.txt") - - delist = [self, self.PanelTop, self.PlotArea] - for item in delist: - if hasattr(item, "analysis"): - del item.analysis - - # check session integrity - messages = analysis.session_check_index(indexfile, search_path=dirname) - - while len(messages["missing tdms"]): - # There are missing tdms files. We need to modify the extracted - # index file with a folder. - missing = messages["missing tdms"] - directories = [] # search directories - updict = {} # new dicts for individual measurements - # Ask user for directory - miss = os.path.basename(missing[0][1]) - - message = _("ShapeOut could not find the following measurements:")+\ - "\n\n".join([""]+[m[1] for m in missing]) +"\n\n"+\ - _("Please select a directory that contains these.") - - dlg = wx.MessageDialog(self, - caption=_("Missing tdms files for session"), - message=message, - style=wx.CANCEL|wx.OK, - ) - mod = dlg.ShowModal() - dlg.Destroy() - if mod != wx.ID_OK: - break - - dlg = wx.DirDialog(self, - message=_( - "Please select directory containing {}" - ).format(miss), - ) - mod = dlg.ShowModal() - path = dlg.GetPath() - dlg.Destroy() - if mod != wx.ID_OK: - break - - # Add search directory - directories.insert(0, path) - - # Try to find all measurements with that directory (also relative) - wx.BeginBusyCursor() - remlist = [] - for m in missing: - key, tdms, thash = m - newfile = tlabwrap.search_hashed_tdms(tdms, thash, directories) - if newfile is not None: - newdir = os.path.dirname(newfile) - updict[key] = {"fdir": newdir} - directories.insert(0, os.path.dirname(newdir)) - directories.insert(0, os.path.dirname(os.path.dirname(newdir))) - remlist.append(m) - for m in remlist: - missing.remove(m) - wx.EndBusyCursor() - - # Update the extracted index file. - analysis.session_update_index(indexfile, updict) - - self.NewAnalysis(indexfile, search_path=dirname) - - directories = list() - for mm in self.analysis.measurements: - if os.path.exists(mm.fdir): - directories.append(mm.fdir) - - bolddirs = self.analysis.GetTDMSFilenames() - self.OnMenuSearchPathAdd(add=False, path=directories, - marked=bolddirs) - - # Remove all temporary files - shutil.rmtree(tempdir, ignore_errors=True) + session.open_session(fname, self) def OnMenuSearchPath(self, e=None): @@ -599,10 +533,13 @@ def OnMenuQuit(self, e=None): if filename is None: # User did not save session - abort return - #self.Close() - #self.Destroy() - #sys.exit() - # Force Exit without cleanup + + # remove the autosaved file + try: + if os.path.exists(autosave.autosave_file): + os.remove(autosave.autosave_file) + except: + pass os._exit(0) @@ -619,20 +556,7 @@ def OnMenuSaveSimple(self, e=None): path += ".zmso" dirname = os.path.dirname(path) self.config.SetWorkingDirectory(dirname, name="Session") - # Begin saving - returnWD = os.getcwd() - tempdir = tempfile.mkdtemp() - os.chdir(tempdir) - Arc = zipfile.ZipFile(path, mode='w') - ## Dump data into directory - self.analysis.DumpData(tempdir, rel_path=os.path.dirname(path)) - for root, _dirs, files in os.walk(tempdir): - for f in files: - fw = os.path.join(root,f) - Arc.write(os.path.relpath(fw,tempdir)) - os.remove(fw) - Arc.close() - os.chdir(returnWD) + session.save_session(path, self.analysis) return path else: dirname = dlg.GetDirectory() diff --git a/shapeout/gui/session.py b/shapeout/gui/session.py new file mode 100644 index 0000000..5334bcf --- /dev/null +++ b/shapeout/gui/session.py @@ -0,0 +1,140 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" ShapeOut - session handling""" +from __future__ import division, print_function + +from dclab.rtdc_dataset import hashfile +import os +import shutil +import tempfile +import wx +import zipfile + +from .. import analysis + + +def open_session(path, parent): + """Open a session file into shapeout + + This method performs a lot of logic on `parent`, the + graphical user interface itself, such as cleanup and + post-processing steps before and after data import. + """ + # Cleanup + delist = [parent, parent.PanelTop, parent.PlotArea] + for item in delist: + if hasattr(item, "analysis"): + del item.analysis + + Arc = zipfile.ZipFile(path, mode='r') + tempdir = tempfile.mkdtemp() + Arc.extractall(tempdir) + Arc.close() + + indexfile = os.path.join(tempdir, "index.txt") + + # check session integrity + dirname = os.path.dirname(path) + messages = analysis.session_check_index(indexfile, search_path=dirname) + + while len(messages["missing tdms"]): + # There are missing tdms files. We need to modify the extracted + # index file with a folder. + missing = messages["missing tdms"] + directories = [] # search directories + updict = {} # new dicts for individual measurements + # Ask user for directory + miss = os.path.basename(missing[0][1]) + + message = _("ShapeOut could not find the following measurements:")+\ + "\n\n".join([""]+[m[1] for m in missing]) +"\n\n"+\ + _("Please select a directory that contains these.") + + dlg = wx.MessageDialog(parent, + caption=_("Missing tdms files for session"), + message=message, + style=wx.CANCEL|wx.OK, + ) + mod = dlg.ShowModal() + dlg.Destroy() + if mod != wx.ID_OK: + break + + dlg = wx.DirDialog(parent, + message=_( + "Please select directory containing {}" + ).format(miss), + ) + mod = dlg.ShowModal() + path = dlg.GetPath() + dlg.Destroy() + if mod != wx.ID_OK: + break + + # Add search directory + directories.insert(0, path) + + # Try to find all measurements with that directory (also relative) + wx.BeginBusyCursor() + remlist = [] + for m in missing: + key, tdms, thash = m + newfile = search_hashed_tdms(tdms, thash, directories) + if newfile is not None: + newdir = os.path.dirname(newfile) + updict[key] = {"fdir": newdir} + directories.insert(0, os.path.dirname(newdir)) + directories.insert(0, os.path.dirname(os.path.dirname(newdir))) + remlist.append(m) + for m in remlist: + missing.remove(m) + wx.EndBusyCursor() + + # Update the extracted index file. + analysis.session_update_index(indexfile, updict) + + parent.NewAnalysis(indexfile, search_path=dirname) + + directories = list() + for mm in parent.analysis.measurements: + if os.path.exists(mm.fdir): + directories.append(mm.fdir) + + bolddirs = parent.analysis.GetTDMSFilenames() + + parent.OnMenuSearchPathAdd(add=False, path=directories, + marked=bolddirs) + + # Remove all temporary files + shutil.rmtree(tempdir, ignore_errors=True) + + +def save_session(path, analysis): + # Begin saving + returnWD = os.getcwd() + tempdir = tempfile.mkdtemp() + os.chdir(tempdir) + with zipfile.ZipFile(path, mode='w') as arc: + ## Dump data into directory + analysis.DumpData(tempdir, rel_path=os.path.dirname(path)) + for root, _dirs, files in os.walk(tempdir): + for f in files: + fw = os.path.join(root,f) + arc.write(os.path.relpath(fw,tempdir)) + os.remove(fw) + os.chdir(returnWD) + + +def search_hashed_tdms(tdms_file, tdms_hash, directories): + """ Search `directories` for `tdms_file` with matching `tdms_hash` + """ + tdms_file = os.path.basename(tdms_file) + for adir in directories: + for root, _ds, fs in os.walk(adir): + if tdms_file in fs: + this_file = os.path.join(root,tdms_file) + this_hash = hashfile(this_file) + if this_hash == tdms_hash: + return this_file + + diff --git a/shapeout/gui/update.py b/shapeout/gui/update.py index b6eab47..6f231d7 100644 --- a/shapeout/gui/update.py +++ b/shapeout/gui/update.py @@ -14,6 +14,7 @@ from .. import _version as so_version + def check_release( ghrepo="user/repo", version=None, timeout=20): @@ -57,6 +58,7 @@ def check_release( msg = a["browser_download_url"] return update, msg + def Update(parent): """ This is a thread for _Update """ @@ -71,6 +73,7 @@ def Update(parent): wargs=(ghrepo, version), cargs=(parent,)) + def _UpdateConsumer(delayedresult, parent): results = delayedresult.get() parent.StatusBar.SetStatusText("Update: "+results[1]) @@ -85,10 +88,9 @@ def get_update(e=None, url=results[1]): webbrowser.open(url) parent.Bind(wx.EVT_MENU, get_update, menudl) - # Do not block GUI too long! - time.sleep(1) parent.StatusBar.SetStatusText("") + def _UpdateWorker(*args): results = check_release(*args) return results diff --git a/shapeout/tlabwrap.py b/shapeout/tlabwrap.py index be57fe7..249b30e 100644 --- a/shapeout/tlabwrap.py +++ b/shapeout/tlabwrap.py @@ -11,26 +11,11 @@ import os import warnings -from dclab.rtdc_dataset import hashfile from dclab import GetTDMSFiles, GetProjectNameFromPath from dclab import config as dc_config - from util import findfile -def search_hashed_tdms(tdms_file, tdms_hash, directories): - """ Search `directories` for `tdms_file` with matching `tdms_hash` - """ - tdms_file = os.path.basename(tdms_file) - for adir in directories: - for root, _ds, fs in os.walk(adir): - if tdms_file in fs: - this_file = os.path.join(root,tdms_file) - this_hash = hashfile(this_file) - if this_hash == tdms_hash: - return this_file - - def crop_linear_data(data, xmin, xmax, ymin, ymax): """ Crop plotting data.