From bd4e99464588dbdbe2dd7cebd8c264b8a604b41f Mon Sep 17 00:00:00 2001 From: theid Date: Thu, 30 Mar 2017 09:04:34 +0200 Subject: [PATCH 01/29] Corsika particles --- km3pipe/io/evt.py | 48 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/km3pipe/io/evt.py b/km3pipe/io/evt.py index 0c2fa81fb..257b63483 100644 --- a/km3pipe/io/evt.py +++ b/km3pipe/io/evt.py @@ -197,6 +197,8 @@ def _create_blob_entry_for_line(self, line, blob): blob.setdefault("EvtRawHits", []).append(raw_hit) if tag == "track_in": blob.setdefault("TrackIns", []).append(TrackIn(values)) + if tag == "track_neutrinos": + blob.setdefault("TrackInNeutrinos", []).append(TrackIn(values)) if tag == "track_fit": blob.setdefault("TrackFits", []).append(TrackFit(values)) if tag == "track_seamuon": @@ -209,10 +211,10 @@ def _create_blob_entry_for_line(self, line, blob): blob['Neutrino'] = Neutrino(values) elif tag == 'center_on_can': values = [float(x) for x in value.split()] - blob['CenterOnCan'] = TrackIn(values) - elif tag == 'primary': + blob['CenterOnCan'] = TrackCorsika(values) + elif tag == 'track_primary': values = [float(x) for x in value.split()] - blob['Primary'] = TrackIn(values) + blob['Primary'] = TrackCorsika(values) else: blob[tag] = value.split() @@ -284,11 +286,40 @@ class TrackIn(Track): """Representation of a track_in entry in an EVT file""" def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) - self.particle_type = geant2pdg(int(self.args[0])) try: - self.length = self.args[1] - except IndexError: + self.particle_type = int(self.args[0]) + self.charmed = bool(self.args[1]) + self.mother = int(self.args[2]) + self.grandmother = int(self.args[3]) self.length = 0 + except IndexError: + self.particle_type = geant2pdg(int(self.args[0])) + try: + self.length = self.args[1] + except IndexError: + self.length = 0 + + def __repr__(self): + text = super(self.__class__, self).__repr__() + text += " length: {0} [m]\n".format(self.length) + try: + text += " type: {0} [Corsika]\n".format(self.particle_type) + text += " charmed: {0}\n".format(self.charmed) + text += " mother: {0} [Corsika]\n".format(self.mother) + text += " grandmother: {0} [Corsika]\n".format(self.grandmother) + except AttributeError: + text += " type: {0} '{1}' [PDG]\n".format(self.particle_type, + pdg2name(self.particle_type)) + pass + + return text + + +class TrackCorsika(Track): + """Representation of a track in a corsika output file""" + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + self.particle_type = int(self.args[0]) try: self.charmed = bool(self.args[1]) self.mother = int(self.args[2]) @@ -298,9 +329,7 @@ def __init__(self, *args, **kwargs): def __repr__(self): text = super(self.__class__, self).__repr__() - text += " type: {0} '{1}' [PDG]\n".format(self.particle_type, - pdg2name(self.particle_type)) - text += " length: {0} [m]\n".format(self.length) + text += " type: {0} [Corsika]\n".format(self.particle_type) try: text += " charmed: {0}\n".format(self.charmed) text += " mother: {0} [Corsika]\n".format(self.mother) @@ -309,7 +338,6 @@ def __repr__(self): pass return text - class TrackFit(Track): """Representation of a track_fit entry in an EVT file""" def __init__(self, *args, **kwargs): From ce5980aa32a5346949e9bf1b06f03ecdd7f34c47 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Mon, 3 Apr 2017 21:48:36 +0200 Subject: [PATCH 02/29] import styles --- km3pipe/{style.py => style/__init__.py} | 7 ------- km3pipe/style/default.py | 1 + km3pipe/style/jonas_phd.py | 3 +++ km3pipe/style/km3pipe.py | 3 +++ km3pipe/style/km3pipe_notebook.py | 3 +++ km3pipe/style/km3pipe_poster.py | 3 +++ km3pipe/style/km3pipe_talk.py | 3 +++ 7 files changed, 16 insertions(+), 7 deletions(-) rename km3pipe/{style.py => style/__init__.py} (96%) create mode 120000 km3pipe/style/default.py create mode 100644 km3pipe/style/jonas_phd.py create mode 100644 km3pipe/style/km3pipe.py create mode 100644 km3pipe/style/km3pipe_notebook.py create mode 100644 km3pipe/style/km3pipe_poster.py create mode 100644 km3pipe/style/km3pipe_talk.py diff --git a/km3pipe/style.py b/km3pipe/style/__init__.py similarity index 96% rename from km3pipe/style.py rename to km3pipe/style/__init__.py index 25161048b..022f4cbba 100644 --- a/km3pipe/style.py +++ b/km3pipe/style/__init__.py @@ -126,10 +126,3 @@ def available(self): def next(self): """Return the next colour in current palette""" return next(self._cycler) - - -# Automatically load default style on import. -try: - use('km3pipe') -except NameError: - print("Could not load default matplotlib style.") diff --git a/km3pipe/style/default.py b/km3pipe/style/default.py new file mode 120000 index 000000000..c6275c0d3 --- /dev/null +++ b/km3pipe/style/default.py @@ -0,0 +1 @@ +km3pipe.py \ No newline at end of file diff --git a/km3pipe/style/jonas_phd.py b/km3pipe/style/jonas_phd.py new file mode 100644 index 000000000..0b14141e7 --- /dev/null +++ b/km3pipe/style/jonas_phd.py @@ -0,0 +1,3 @@ +from ..style import use + +use('jonas-phd') diff --git a/km3pipe/style/km3pipe.py b/km3pipe/style/km3pipe.py new file mode 100644 index 000000000..327fdc54f --- /dev/null +++ b/km3pipe/style/km3pipe.py @@ -0,0 +1,3 @@ +from ..style import use + +use('km3pipe') diff --git a/km3pipe/style/km3pipe_notebook.py b/km3pipe/style/km3pipe_notebook.py new file mode 100644 index 000000000..235994377 --- /dev/null +++ b/km3pipe/style/km3pipe_notebook.py @@ -0,0 +1,3 @@ +from ..style import use + +use('km3pipe-notebook') diff --git a/km3pipe/style/km3pipe_poster.py b/km3pipe/style/km3pipe_poster.py new file mode 100644 index 000000000..665b11874 --- /dev/null +++ b/km3pipe/style/km3pipe_poster.py @@ -0,0 +1,3 @@ +from ..style import use + +use('km3pipe-poster') diff --git a/km3pipe/style/km3pipe_talk.py b/km3pipe/style/km3pipe_talk.py new file mode 100644 index 000000000..61f1ce4ac --- /dev/null +++ b/km3pipe/style/km3pipe_talk.py @@ -0,0 +1,3 @@ +from ..style import use + +use('km3pipe-talk') From e2dbb9428efc4920744cb67cd62786fab2f8da20 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Tue, 4 Apr 2017 11:49:21 +0200 Subject: [PATCH 03/29] rtd theme --- docs/_static/custom.css | 71 ----------------------------------------- docs/conf.py | 11 +++---- sphinx_requirements.txt | 1 + 3 files changed, 6 insertions(+), 77 deletions(-) delete mode 100644 docs/_static/custom.css diff --git a/docs/_static/custom.css b/docs/_static/custom.css deleted file mode 100644 index b72fa1b7f..000000000 --- a/docs/_static/custom.css +++ /dev/null @@ -1,71 +0,0 @@ -@import 'https://fonts.googleapis.com/css?family=Lato:400,700,900|Open+Sans|Roboto+Mono'; - -pre, code { - font-family: "Roboto Mono", monospace; - font-size: 14px; -} -pre { - line-height: 1.4em; - overflow-x: auto; -} - -code, tt { - padding: 1px 3px; - font-family: "Roboto Mono", monospace; - font-size: 0.85em; - white-space: pre-wrap; -} - -pre { - margin: 0 0 1.75em 0; - width: 100%; - padding: 10px; - font-family: "Roboto Mono", monospace; - font-size: 0.75em; - white-space: pre; - overflow: auto; -} - -pre code, pre tt { - font-size: inherit; - white-space: pre-wrap; - border: none; - padding: 0; -} - -/* better styling for tables, i.e. pd.DataFrame */ - -table { - border: none; - border-collapse: collapse; - border-spacing: 0; - table-layout: fixed; - margin-left: auto; - margin-right: auto; -} - -thead { - vertical-align: bottom; - border-bottom: 1px solid; - text-align: right; -} - -/* alabaster has some weird div.body table defines, - * and I can't override them by `td, th` or - * `.dataframe td`. WTF. -div.body td, div.body th { - text-align: right; - vertical-align: right; - padding: 0.5em 0.5em; - line-height: 1.0; - white-space: nowrap; - max-width: 100px; - text-overflow: ellipsis; - overflow: hidden; - border: none; -} - */ - -div.sphinxsidebar ul li { - line-height: 1.8; -} diff --git a/docs/conf.py b/docs/conf.py index 99cadd924..bdfeb44a9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,6 +15,7 @@ import sys import os from datetime import date +import sphinx_rtd_theme # what is this? #sys.path.append('../') @@ -112,12 +113,10 @@ # -- Options for HTML output ---------------------------------------------- -html_theme = 'alabaster' -html_theme_options = { - 'font_family': "'Open Sans', sans", - 'head_font_family': "'Lato', sans", - 'code_font_family': "'Roboto Mono', monospace", -} +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +#html_theme_options = { +#} # The name for this set of Sphinx documents. If None, it defaults to diff --git a/sphinx_requirements.txt b/sphinx_requirements.txt index be43a5e07..eb30074ae 100644 --- a/sphinx_requirements.txt +++ b/sphinx_requirements.txt @@ -7,3 +7,4 @@ sphinxcontrib-napoleon scikit-learn statsmodels==0.8.0rc1 seaborn +sphinx_rtd_theme From 1ad07e0f5f1461748045f71beee136bb4d39be87 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Tue, 4 Apr 2017 13:51:03 +0200 Subject: [PATCH 04/29] Shameless Self Promotion --- km3pipe/kp-data/stylelib/moritz.mplstyle | 15 +++++++++++++++ km3pipe/style/moritz.py | 3 +++ 2 files changed, 18 insertions(+) create mode 100644 km3pipe/kp-data/stylelib/moritz.mplstyle create mode 100644 km3pipe/style/moritz.py diff --git a/km3pipe/kp-data/stylelib/moritz.mplstyle b/km3pipe/kp-data/stylelib/moritz.mplstyle new file mode 100644 index 000000000..c1a13910a --- /dev/null +++ b/km3pipe/kp-data/stylelib/moritz.mplstyle @@ -0,0 +1,15 @@ +axes.linewidth: 2 + +image.cmap: viridis + +font.sans-serif: Roboto, sans-serif +font.serif: Roboto Slab, serif +font.monospace: Roboto Mono, serif + +hist.bins: auto + +savefig.bbox: tight +savefig.format: svg +savefig.transparent: false + +svg.fonttype: none diff --git a/km3pipe/style/moritz.py b/km3pipe/style/moritz.py new file mode 100644 index 000000000..453d748da --- /dev/null +++ b/km3pipe/style/moritz.py @@ -0,0 +1,3 @@ +from ..style import use + +use('moritz') From 7475ca22571c36297788d59a60ca22d6692d4308 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Thu, 6 Apr 2017 13:20:30 +0200 Subject: [PATCH 05/29] Pass silently when update-check fails: --- km3pipe/__version__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/km3pipe/__version__.py b/km3pipe/__version__.py index 3c5fb8031..275253dfa 100644 --- a/km3pipe/__version__.py +++ b/km3pipe/__version__.py @@ -73,4 +73,7 @@ def check_for_update(): version = _get_version(version_info) if config is not None: if config.check_for_updates: - check_for_update() + try: + check_for_update() + except (ValueError, TypeError, OSError): + pass From f195847c6200e14e7af9bb0bc472c723dab86897 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Apr 2017 23:06:09 +0200 Subject: [PATCH 06/29] Add totmonitor --- km3pipe/utils/totmonitor.py | 125 ++++++++++++++++++++++++++++++++++++ setup.py | 1 + 2 files changed, 126 insertions(+) create mode 100755 km3pipe/utils/totmonitor.py diff --git a/km3pipe/utils/totmonitor.py b/km3pipe/utils/totmonitor.py new file mode 100755 index 000000000..f75ebaf6c --- /dev/null +++ b/km3pipe/utils/totmonitor.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# coding=utf-8 +# Filename: totmonitor.py +""" +Display the mean ToT for each TDC channel for given DOM. + +Usage: + totmonitor [-l LIGIER] [-p PORT] [-o OPTIMAL_TOT] [-t tolerance] DOM_ID + totmonitor (-h | --help) + totmonitor --version + +Options: + -l LIGIER Ligier address [default: 127.0.0.1]. + -p PORT Ligier port [default: 5553]. + -o OPTIMAL_TOT Target ToT in ns [default: 26.4]. + -t TOLERANCE Defines the range for valid ToT [default: 0.3]. + -h --help Show this screen. +""" +from __future__ import division + +import os +from collections import defaultdict + +from termcolor import colored, cprint + +import km3pipe as kp +import numpy as np + +__author__ = "Tamas Gal and Jonas Reubelt" +__copyright__ = "Copyright 2017, the KM3NeT collaboration." +__credits__ = [] +__license__ = "MIT" +__maintainer__ = "Tamas Gal and Jonas Reubelt" +__email__ = "tgal@km3net.de" +__status__ = "Development" + + +class TimesliceCreator(kp.core.Module): + """Create `TimesliceHitSeries` from raw timeslice hits.""" + def configure(self): + self.dom_id = self.require("dom_id") + + def process(self, blob): + hits = blob['TimesliceFrames'][self.dom_id] + n_hits = len(hits) + channel_ids, times, tots = zip(*hits) + ts_hits = kp.dataclasses.TimesliceHitSeries.from_arrays( + np.array(channel_ids), + np.full(n_hits, self.dom_id), + np.array(times), + np.array(tots), + 0, + 0) + blob['TimesliceHits'] = ts_hits + return blob + + +class MeanTotDisplay(kp.core.Module): + def configure(self): + self.tots = defaultdict(list) + self.counter = 0 + + def process(self, blob): + hits = blob["TimesliceHits"] + for channel in range(31): + idx = hits.channel_id == channel + tots = hits.tot[idx] + self.tots[channel] += list(tots) + + self.counter += 1 + if self.counter % 100 == 0: + self.update_display() + self.tots = defaultdict(list) + self.counter = 0 + return blob + + def update_display(self): + os.system('clear') + self.print_header() + for channel, tots in self.tots.iteritems(): + if channel % 8 == 0: + self.print_scale() + mean_tot = np.mean(tots) + if np.isnan(mean_tot): + mean_tot = 0 + color = 'green' + if mean_tot > OPTIMAL_TOT + TOLERANCE: + color = 'red' + if mean_tot < OPTIMAL_TOT - TOLERANCE: + color = 'blue' + cprint("Channel {0:02d}: {1:.1f}ns {2}" + .format(channel, mean_tot, int(mean_tot) * '|'), + color) + self.print_scale() + self.print_header() + + def print_header(self): + print(" " + "0 10 20 30 40 50") + + def print_scale(self): + print(" " + '|----+----' * 10) + + +def main(): + from docopt import docopt + args = docopt(__doc__) + print(args) + pipe = kp.Pipeline() + pipe.attach(kp.io.ch.CHPump, + host=args['-l'], + port=int(args['-p']), + tags='IO_TSL', + max_queue=100, + timeout=60*60*24) + pipe.attach(kp.io.daq.TimesliceParser) + pipe.attach(TimesliceCreator, dom_id=int(args['DOM_ID'])) + pipe.attach(MeanTotDisplay, + optimal_tot=float(args['-o']), + tolreance=float(args['-t'])) + pipe.drain() + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index beb2570ba..b147c3831 100644 --- a/setup.py +++ b/setup.py @@ -123,6 +123,7 @@ def finalize_options(self): 'pushover=km3pipe.utils.pushover:main', 'ztplot=km3pipe.utils.ztplot:main', 'k40calib=km3pipe.utils.k40calib:main', + 'totmonitor=km3pipe.utils.totmonitor:main', ], }, classifiers=[ From 4f93a7fcc9016b06f8a3243857db20ef3c866131 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Apr 2017 23:10:52 +0200 Subject: [PATCH 07/29] Fix typo --- km3pipe/utils/pushover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/km3pipe/utils/pushover.py b/km3pipe/utils/pushover.py index f6d43994a..f47500e65 100644 --- a/km3pipe/utils/pushover.py +++ b/km3pipe/utils/pushover.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Filename: tohdf5.py +# Filename: pushover.py """ Send a push message to a device using Pushover.net API. From fe6ce18737cf141f5226b5dee83f37f891aec9d6 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Apr 2017 23:20:38 +0200 Subject: [PATCH 08/29] Include style dir --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 2e6bde3fc..3a13606db 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ recursive-include km3pipe *.pyxbld recursive-include docs * recursive-include km3pipe/tests *.py recursive-include km3pipe/io/tests *.py +recursive-include km3pipe/style *.py recursive-include km3pipe/kp-data * recursive-exclude docs *.pyc From dabcea12e26eff7671c68b24aca52a5612a81f10 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Apr 2017 23:27:09 +0200 Subject: [PATCH 09/29] Fix typo and remove debug messages --- km3pipe/utils/totmonitor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/km3pipe/utils/totmonitor.py b/km3pipe/utils/totmonitor.py index f75ebaf6c..21f20ec7f 100755 --- a/km3pipe/utils/totmonitor.py +++ b/km3pipe/utils/totmonitor.py @@ -84,9 +84,9 @@ def update_display(self): if np.isnan(mean_tot): mean_tot = 0 color = 'green' - if mean_tot > OPTIMAL_TOT + TOLERANCE: + if mean_tot > self.optimal_tot + self.tolreance: color = 'red' - if mean_tot < OPTIMAL_TOT - TOLERANCE: + if mean_tot < self.optimal_tot - self.tolreance: color = 'blue' cprint("Channel {0:02d}: {1:.1f}ns {2}" .format(channel, mean_tot, int(mean_tot) * '|'), @@ -105,7 +105,6 @@ def print_scale(self): def main(): from docopt import docopt args = docopt(__doc__) - print(args) pipe = kp.Pipeline() pipe.attach(kp.io.ch.CHPump, host=args['-l'], From cf6514381cc059ca79924c87cf969b5a28823c80 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Apr 2017 23:34:09 +0200 Subject: [PATCH 10/29] Fix typos --- km3pipe/utils/totmonitor.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/km3pipe/utils/totmonitor.py b/km3pipe/utils/totmonitor.py index 21f20ec7f..1b7433ff0 100755 --- a/km3pipe/utils/totmonitor.py +++ b/km3pipe/utils/totmonitor.py @@ -5,7 +5,7 @@ Display the mean ToT for each TDC channel for given DOM. Usage: - totmonitor [-l LIGIER] [-p PORT] [-o OPTIMAL_TOT] [-t tolerance] DOM_ID + totmonitor [-l LIGIER] [-p PORT] [-o OPTIMAL_TOT] [-t TOLERANCE] [-u UPDATE] DOM_ID totmonitor (-h | --help) totmonitor --version @@ -14,6 +14,7 @@ -p PORT Ligier port [default: 5553]. -o OPTIMAL_TOT Target ToT in ns [default: 26.4]. -t TOLERANCE Defines the range for valid ToT [default: 0.3]. + -u UPDATE Update frequency in seconds [default: 10]. -h --help Show this screen. """ from __future__ import division @@ -57,6 +58,9 @@ def process(self, blob): class MeanTotDisplay(kp.core.Module): def configure(self): + self.optimal_tot = self.get("optimal_tot") or 26.4 + self.tolerance = self.get("tolerance") or 0.3 + self.update_frequency = self.get("update_frequency") or 10 self.tots = defaultdict(list) self.counter = 0 @@ -68,7 +72,7 @@ def process(self, blob): self.tots[channel] += list(tots) self.counter += 1 - if self.counter % 100 == 0: + if self.counter % int(self.update_frequency * 10) == 0: self.update_display() self.tots = defaultdict(list) self.counter = 0 @@ -84,9 +88,9 @@ def update_display(self): if np.isnan(mean_tot): mean_tot = 0 color = 'green' - if mean_tot > self.optimal_tot + self.tolreance: + if mean_tot > self.optimal_tot + self.tolerance: color = 'red' - if mean_tot < self.optimal_tot - self.tolreance: + if mean_tot < self.optimal_tot - self.tolerance: color = 'blue' cprint("Channel {0:02d}: {1:.1f}ns {2}" .format(channel, mean_tot, int(mean_tot) * '|'), @@ -116,7 +120,8 @@ def main(): pipe.attach(TimesliceCreator, dom_id=int(args['DOM_ID'])) pipe.attach(MeanTotDisplay, optimal_tot=float(args['-o']), - tolreance=float(args['-t'])) + update_frequency=float(args['-u']), + tolerance=float(args['-t'])) pipe.drain() From fd9b7d90df4d14eee9be5f43909762fa8fd350a5 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Wed, 12 Apr 2017 23:38:12 +0200 Subject: [PATCH 11/29] Skip 0 hits events --- km3pipe/utils/totmonitor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/km3pipe/utils/totmonitor.py b/km3pipe/utils/totmonitor.py index 1b7433ff0..330a7b7b2 100755 --- a/km3pipe/utils/totmonitor.py +++ b/km3pipe/utils/totmonitor.py @@ -44,6 +44,8 @@ def configure(self): def process(self, blob): hits = blob['TimesliceFrames'][self.dom_id] n_hits = len(hits) + if n_hits == 0: + return blob channel_ids, times, tots = zip(*hits) ts_hits = kp.dataclasses.TimesliceHitSeries.from_arrays( np.array(channel_ids), @@ -119,6 +121,7 @@ def main(): pipe.attach(kp.io.daq.TimesliceParser) pipe.attach(TimesliceCreator, dom_id=int(args['DOM_ID'])) pipe.attach(MeanTotDisplay, + only_if="TimesliceHits", optimal_tot=float(args['-o']), update_frequency=float(args['-u']), tolerance=float(args['-t'])) From 334f250698f65a9778f61e55efb14fc39fe58a7c Mon Sep 17 00:00:00 2001 From: theid Date: Wed, 19 Apr 2017 09:54:04 +0200 Subject: [PATCH 12/29] corrected doc string to Detector Module --- km3pipe/core.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/km3pipe/core.pyx b/km3pipe/core.pyx index 336511128..1cac78c28 100644 --- a/km3pipe/core.pyx +++ b/km3pipe/core.pyx @@ -386,7 +386,7 @@ class Geometry(Module): Parameters ---------- - should_apply: bool, optional [default=False] + apply: bool, optional [default=False] Apply the geometry to the hits (add position/direction/t0)? filename: str, optional [default=None] DetX file with detector description. From 1530cfff1c11a71a6390fdd3cc2f9d37592064fa Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Fri, 21 Apr 2017 13:05:35 +0200 Subject: [PATCH 13/29] More debug messages --- km3pipe/db.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/km3pipe/db.py b/km3pipe/db.py index 7d394b91f..7e0d47d95 100644 --- a/km3pipe/db.py +++ b/km3pipe/db.py @@ -171,6 +171,7 @@ def _add_datetime(self, dataframe, timestamp_key='UNIXTIME'): def convert_data(timestamp): return datetime.fromtimestamp(float(timestamp) / 1e3, UTC_TZ) try: + log.debug("Adding DATETIME column to the data") converted = dataframe[timestamp_key].apply(convert_data) dataframe['DATETIME'] = converted except KeyError: @@ -180,6 +181,7 @@ def _add_converted_units(self, dataframe, parameter, key='VALUE'): """Add an additional DATA_VALUE column with converted VALUEs""" convert_unit = self.parameters.get_converter(parameter) try: + log.debug("Adding unit converted DATA_VALUE to the data") dataframe[key] = dataframe['DATA_VALUE'].apply(convert_unit) except KeyError: log.warn("Missing 'VALUE': no unit conversion.") @@ -322,15 +324,23 @@ def _get_content(self, url): def opener(self): "A reusable connection manager" if self._opener is None: + log.debug("Creating connection handler") opener = build_opener() + if self._cookies: + log.debug("Appending cookies") + else: + log.debug("No cookies to append") for cookie in self._cookies: cookie_str = cookie.name + '=' + cookie.value opener.addheaders.append(('Cookie', cookie_str)) self._opener = opener + else: + log.debug("Reusing connection manager") return self._opener def request_sid_cookie(self, username, password): """Request cookie for permanent session token.""" + log.debug("Requesting SID cookie") target_url = self._login_url + '?usr={0}&pwd={1}&persist=y' \ .format(username, password) cookie = urlopen(target_url).read() @@ -338,13 +348,16 @@ def request_sid_cookie(self, username, password): def restore_session(self, cookie): """Establish databse connection using permanent session cookie""" + log.debug("Restoring session from cookie") opener = build_opener() opener.addheaders.append(('Cookie', cookie)) self._opener = opener def request_permanent_session(self, username=None, password=None): + log.debug("Requesting permanent session") config = Config() if username is None and password is None: + log.debug("Checking configuration file for DB credentials") username, password = config.db_credentials cookie = self.request_sid_cookie(username, password) try: @@ -352,15 +365,18 @@ def request_permanent_session(self, username=None, password=None): except TypeError: cookie_str = str(cookie) # Python 2 log.debug("Session cookie: {0}".format(cookie_str)) + log.debug("Storing cookie in configuration file") config.set('DB', 'session_cookie', cookie_str) self.restore_session(cookie) def login(self, username, password): - "Login to the databse and store cookies for upcoming requests." + "Login to the database and store cookies for upcoming requests." + log.debug("Logging in to the DB") opener = self._build_opener() values = {'usr': username, 'pwd': password} req = self._make_request(self._login_url, values) try: + log.debug("Sending login request") f = opener.open(req) except URLError as e: log.error("Failed to connect to the database -> probably down!") From e0b013de28ff032c60bf3d2f754fd4dad77d5f39 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Fri, 21 Apr 2017 13:28:07 +0200 Subject: [PATCH 14/29] Logging docs --- docs/index.rst | 1 + docs/logging.rst | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 docs/logging.rst diff --git a/docs/index.rst b/docs/index.rst index 8795f42e2..fec898eda 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,7 @@ install user_guide + logging examples api contribute diff --git a/docs/logging.rst b/docs/logging.rst new file mode 100644 index 000000000..ec26517fe --- /dev/null +++ b/docs/logging.rst @@ -0,0 +1,49 @@ +Logging +======= + + +.. contents:: :local: + + +Introduction +------------ +KM3Pipe uses a module based logging system which can individually be configured to +provide information needed to understand the underlying mechanisms. + +You can also easily create your own logger. + + +Accessing a Logger +------------------ +To access a modules logger, you need to:: + + from km3pipe.logger import logging + log = logging.getLogger("module.path") + +where ``module.path`` is the Python import path of the module, like ``km3pipe.core``. + +To set a desired logging level, use the keywords ``DEBUG``, ``INFO``, ``WARNING``, +``ERROR`` or ``CRITICAL``. For example:: + + log.setLevel("DEBUG") + +Creating your own Logger +------------------------ + +To create your own logger, use the same procedure as described above:: + + from km3pipe.logger import logging + log = logging.getLogger("your.desired.logger.name") + +After that, you can use it to log anywhere:: + + log.debug("A debug message") + log.info("An info message") + log.warn("A warning") + log.error("An error") + log.critical("A critical think") + +and set its debug level:: + + log.setLevel("WARN") + From 434c2c827b44e2d22edf4f089274314ee5b0e39a Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Fri, 21 Apr 2017 20:38:27 +0200 Subject: [PATCH 15/29] Migrate termcolor/MIT --- km3pipe/dev.py | 104 +++++++++++++++++++++++++++++++++++- km3pipe/utils/totmonitor.py | 4 +- 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/km3pipe/dev.py b/km3pipe/dev.py index a85e9afcd..4a5e607c9 100644 --- a/km3pipe/dev.py +++ b/km3pipe/dev.py @@ -11,14 +11,14 @@ import collections import re +import os import warnings - from .logger import logging __author__ = "Tamas Gal and Moritz Lotze" __copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration." -__credits__ = [] +__credits__ = ["Konstantin Lepa for termcolor"] __license__ = "MIT" __maintainer__ = "Tamas Gal and Moritz Lotze" __email__ = "tgal@km3net.de" @@ -204,3 +204,103 @@ def _update_doc(self, olddoc): if olddoc: newdoc = "%s\n\n%s" % (newdoc, olddoc) return newdoc + + +ATTRIBUTES = dict( + list(zip([ + 'bold', + 'dark', + '', + 'underline', + 'blink', + '', + 'reverse', + 'concealed' + ], + list(range(1, 9)) + )) + ) +del ATTRIBUTES[''] + +ATTRIBUTES_RE = '\033\[(?:%s)m' % '|' \ + .join(['%d' % v for v in ATTRIBUTES.values()]) + +HIGHLIGHTS = dict( + list(zip([ + 'on_grey', + 'on_red', + 'on_green', + 'on_yellow', + 'on_blue', + 'on_magenta', + 'on_cyan', + 'on_white' + ], + list(range(40, 48)) + )) + ) + +HIGHLIGHTS_RE = '\033\[(?:%s)m' % '|' \ + .join(['%d' % v for v in HIGHLIGHTS.values()]) + +COLORS = dict( + list(zip([ + 'grey', + 'red', + 'green', + 'yellow', + 'blue', + 'magenta', + 'cyan', + 'white', + ], + list(range(30, 38)) + )) + ) + +COLORS_RE = '\033\[(?:%s)m' % '|'.join(['%d' % v for v in COLORS.values()]) + +RESET = '\033[0m' +RESET_RE = '\033\[0m' + + +def colored(text, color=None, on_color=None, attrs=None): + """Colorize text, while stripping nested ANSI color sequences. + + Author: Konstantin Lepa / termcolor + + Available text colors: + red, green, yellow, blue, magenta, cyan, white. + Available text highlights: + on_red, on_green, on_yellow, on_blue, on_magenta, on_cyan, on_white. + Available attributes: + bold, dark, underline, blink, reverse, concealed. + Example: + colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) + colored('Hello, World!', 'green') + """ + if os.getenv('ANSI_COLORS_DISABLED') is None: + fmt_str = '\033[%dm%s' + if color is not None: + text = re.sub(COLORS_RE + '(.*?)' + RESET_RE, r'\1', text) + text = fmt_str % (COLORS[color], text) + if on_color is not None: + text = re.sub(HIGHLIGHTS_RE + '(.*?)' + RESET_RE, r'\1', text) + text = fmt_str % (HIGHLIGHTS[on_color], text) + if attrs is not None: + text = re.sub(ATTRIBUTES_RE + '(.*?)' + RESET_RE, r'\1', text) + for attr in attrs: + text = fmt_str % (ATTRIBUTES[attr], text) + return text + RESET + else: + return text + + +def cprint(text, color=None, on_color=None, attrs=None, **kwargs): + """Print colorize text. + + Author: Konstantin Lepa / termcolor + + It accepts arguments of print function. + """ + print((colored(text, color, on_color, attrs)), **kwargs) diff --git a/km3pipe/utils/totmonitor.py b/km3pipe/utils/totmonitor.py index 330a7b7b2..2292de8e4 100755 --- a/km3pipe/utils/totmonitor.py +++ b/km3pipe/utils/totmonitor.py @@ -22,7 +22,7 @@ import os from collections import defaultdict -from termcolor import colored, cprint +from km3pipe.dev import cprint import km3pipe as kp import numpy as np @@ -60,7 +60,7 @@ def process(self, blob): class MeanTotDisplay(kp.core.Module): def configure(self): - self.optimal_tot = self.get("optimal_tot") or 26.4 + self.optimal_tot = self.get("optimal_tot") or 26.4 self.tolerance = self.get("tolerance") or 0.3 self.update_frequency = self.get("update_frequency") or 10 self.tots = defaultdict(list) From 6a3e1fcf95c40539cdd3b3d4a89d506c9532895d Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Fri, 21 Apr 2017 21:21:44 +0200 Subject: [PATCH 16/29] Update --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eeb85b8ae..40aff7a1d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,6 @@ Unreleased changes ------------------ +* ``totmonitor`` command line utility added 6.6.6 / 2017-04-03 ------------------ From 095aa7952b8b91372c3d1e639cf8d8ce84dc0b62 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Fri, 21 Apr 2017 22:29:04 +0200 Subject: [PATCH 17/29] Update statsmodels requirement --- sphinx_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx_requirements.txt b/sphinx_requirements.txt index eb30074ae..5f76143f1 100644 --- a/sphinx_requirements.txt +++ b/sphinx_requirements.txt @@ -5,6 +5,6 @@ sphinx-gallery sphinx>=1.5.1 sphinxcontrib-napoleon scikit-learn -statsmodels==0.8.0rc1 +statsmodels>=0.8 seaborn sphinx_rtd_theme From 3dd1140f07de25882f6719845103fe5890597767 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Mon, 24 Apr 2017 17:11:29 +0200 Subject: [PATCH 18/29] bump scipy --- .rtd-environment.yml | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.rtd-environment.yml b/.rtd-environment.yml index af170700c..de14ca8e4 100644 --- a/.rtd-environment.yml +++ b/.rtd-environment.yml @@ -11,7 +11,7 @@ dependencies: - numpy - pandas - matplotlib - - scipy + - scipy>=0.19 - cython - pillow - pytables diff --git a/requirements.txt b/requirements.txt index 09f1baa8c..d6997a981 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cython docopt matplotlib>=2.0.0 pandas -scipy +scipy>=0.19 tables tornado urwid diff --git a/setup.py b/setup.py index b147c3831..dc58df9cf 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ def finalize_options(self): 'base': ['cython', 'docopt', 'numpy>=1.12', 'pandas', 'pytz', 'six', ], 'analysis': ['matplotlib>=2.0.0', 'sklearn', 'statsmodels>=0.8', - 'scipy', 'seaborn', 'ipython', 'patsy', ], + 'scipy>=0.19', 'seaborn', 'ipython', 'patsy', ], 'daq': ['controlhost', ], 'io': ['tables', 'h5py', ], 'jpp': ['jppy>=1.3.1', ], From 1ff7da4438d2b5fe978945c15dc1c5e618631be4 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Tue, 25 Apr 2017 08:49:25 +0200 Subject: [PATCH 19/29] ignore test output --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5fcca03a0..7726e1c0c 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,6 @@ docs/api # jupyter files .ipynb_checkpoints/ + +# +junit.xml From b3ea874484333034bfbc03b99bfb14b9a866b626 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Tue, 25 Apr 2017 08:49:27 +0200 Subject: [PATCH 20/29] bump pytables --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d6997a981..7f32f67d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ docopt matplotlib>=2.0.0 pandas scipy>=0.19 -tables +tables>=3.4 tornado urwid websocket-client diff --git a/setup.py b/setup.py index dc58df9cf..7c80af2de 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ def finalize_options(self): 'analysis': ['matplotlib>=2.0.0', 'sklearn', 'statsmodels>=0.8', 'scipy>=0.19', 'seaborn', 'ipython', 'patsy', ], 'daq': ['controlhost', ], - 'io': ['tables', 'h5py', ], + 'io': ['tables>=3.4', 'h5py', ], 'jpp': ['jppy>=1.3.1', ], 'web': ['tornado', 'websocket-client', ], 'testing': ['pytest', 'mock', ], From a7d80cdf10386b58db6bf40136019c1d1b43c841 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Tue, 25 Apr 2017 09:40:46 +0200 Subject: [PATCH 21/29] Freeze PyTables to 3.4.0 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 09f1baa8c..ac0476102 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ docopt matplotlib>=2.0.0 pandas scipy -tables +tables=3.4.0 tornado urwid websocket-client diff --git a/setup.py b/setup.py index b147c3831..9916bd4fb 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ def finalize_options(self): 'analysis': ['matplotlib>=2.0.0', 'sklearn', 'statsmodels>=0.8', 'scipy', 'seaborn', 'ipython', 'patsy', ], 'daq': ['controlhost', ], - 'io': ['tables', 'h5py', ], + 'io': ['tables==3.4.0', 'h5py', ], 'jpp': ['jppy>=1.3.1', ], 'web': ['tornado', 'websocket-client', ], 'testing': ['pytest', 'mock', ], From 3b2570d3d60178a43f6aa890f15d57a502e0fdf0 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Tue, 25 Apr 2017 09:59:16 +0200 Subject: [PATCH 22/29] Fix typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c6acded3d..d30158789 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ def finalize_options(self): 'analysis': ['matplotlib>=2.0.0', 'sklearn', 'statsmodels>=0.8', 'scipy>=0.19', 'seaborn', 'ipython', 'patsy', ], 'daq': ['controlhost', ], - 'io': ['tables==3.4.0', 'h5py', ], + 'io': ['tables=3.4.0', 'h5py', ], 'jpp': ['jppy>=1.3.1', ], 'web': ['tornado', 'websocket-client', ], 'testing': ['pytest', 'mock', ], From dc44ca76b5f752ff16ea74c9ba71c09158317841 Mon Sep 17 00:00:00 2001 From: Tamas Gal Date: Tue, 25 Apr 2017 10:05:20 +0200 Subject: [PATCH 23/29] I am an idiot --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 70329b968..e1561dd62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cython docopt matplotlib>=2.0.0 pandas -tables=3.4.0 +tables==3.4.0 scipy>=0.19 tornado urwid diff --git a/setup.py b/setup.py index d30158789..c6acded3d 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ def finalize_options(self): 'analysis': ['matplotlib>=2.0.0', 'sklearn', 'statsmodels>=0.8', 'scipy>=0.19', 'seaborn', 'ipython', 'patsy', ], 'daq': ['controlhost', ], - 'io': ['tables=3.4.0', 'h5py', ], + 'io': ['tables==3.4.0', 'h5py', ], 'jpp': ['jppy>=1.3.1', ], 'web': ['tornado', 'websocket-client', ], 'testing': ['pytest', 'mock', ], From fddc327c1c43b3c0dbd8c2863bf0de7b424e0b84 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Mon, 1 May 2017 14:03:08 +0200 Subject: [PATCH 24/29] df_to_h5: add --force flag --- km3pipe/io/pandas.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/km3pipe/io/pandas.py b/km3pipe/io/pandas.py index da356a284..afc09af45 100644 --- a/km3pipe/io/pandas.py +++ b/km3pipe/io/pandas.py @@ -124,17 +124,17 @@ def merge_event_ids(df): return df -def df_to_h5(df, h5file, where): +def df_to_h5(df, h5file, where, **kwargs): """Write pandas dataframes with proper columns. Example: >>> df = pd.DataFrame(...) >>> df_to_h5(df, 'foo.h5', '/some/loc/my_df') """ - write_table(df.to_records(index=False), h5file, where) + write_table(df.to_records(index=False), h5file, where, **kwargs) -def write_table(array, h5file, where): +def write_table(array, h5file, where, force=False): """Write a structured numpy array into a H5 table. """ own_h5 = False @@ -145,8 +145,11 @@ def write_table(array, h5file, where): loc, tabname = os.path.split(where) if loc == '': loc = '/' - h5file.create_table(loc, tabname, obj=array, createparents=True, - filters=filt) + try: + h5file.create_table(loc, tabname, obj=array, createparents=True, + filters=filt) + except tb.exceptions.NodeError: + h5file.get_node(where)[:] = array if own_h5: h5file.close() From 835f04f3c5586b427c8b3b28ccb2b52aaf1aa6b7 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Mon, 8 May 2017 09:32:20 +0200 Subject: [PATCH 25/29] fix pytables type error for some nodes --- km3pipe/io/pandas.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/km3pipe/io/pandas.py b/km3pipe/io/pandas.py index afc09af45..1d6b910e8 100644 --- a/km3pipe/io/pandas.py +++ b/km3pipe/io/pandas.py @@ -54,7 +54,7 @@ class H5Chain(object): def __init__(self, filenames): self.h5files = {} for fn in filenames: - h5 = pd.HDFStore(fn, 'r') + h5 = tb.File(fn, 'r') self.h5files[fn] = h5 def close(self): @@ -70,7 +70,8 @@ def __enter__(self): def __getitem__(self, key): dfs = [] for fname, h5 in self.h5files.items(): - df = h5[key] + tab = h5.get_node(key)[:] + df = pd.DataFrame(tab) dfs.append(df) dfs = pd.concat(dfs, axis=0, ignore_index=True) return dfs From df24322197ee1ccb518af76eb5557badfe97b9b7 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Mon, 8 May 2017 09:46:45 +0200 Subject: [PATCH 26/29] bump changelog --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 40aff7a1d..4282ac16e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,10 @@ Unreleased changes ------------------ + +6.7.0 / 2017-05-08 +------------------ * ``totmonitor`` command line utility added +* bump library versions (scipy >=0.19) 6.6.6 / 2017-04-03 ------------------ From cd9970fc22da46bf675ca885bbd6a4a1d78e105a Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Mon, 8 May 2017 09:47:06 +0200 Subject: [PATCH 27/29] Bumps version number --- km3pipe/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/km3pipe/__version__.py b/km3pipe/__version__.py index 6c317d636..931765d6e 100644 --- a/km3pipe/__version__.py +++ b/km3pipe/__version__.py @@ -28,7 +28,7 @@ log = logging.getLogger(__name__) # pylint: disable=C0103 -version_info = (6, 6, 6, 'final', 0) +version_info = (6, 7, 0, 'final', 0) def _get_version(version_info): From 376ddb8024eec4f4fa7199d2f6dfad9c1fe7073e Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Mon, 8 May 2017 09:47:06 +0200 Subject: [PATCH 28/29] update version tag in docs --- docs/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/version.txt b/docs/version.txt index f480828d3..b33f738df 100755 --- a/docs/version.txt +++ b/docs/version.txt @@ -1,2 +1,2 @@ -KM3Pipe 6.6.6 +KM3Pipe 6.7.0 ============= From 24bdae6a87556381639399ff933ea21577c263a2 Mon Sep 17 00:00:00 2001 From: Moritz Lotze Date: Mon, 8 May 2017 09:47:12 +0200 Subject: [PATCH 29/29] Bumps changelog --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4282ac16e..0bcc31665 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,7 @@ Unreleased changes ------------------ + 6.7.0 / 2017-05-08 ------------------ * ``totmonitor`` command line utility added