From 3cd06bc892045fe2d01f53894b04938fc56d5a4b Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Mon, 18 Jan 2021 19:35:51 -0600 Subject: [PATCH 01/46] Added ship_tracking AIS example --- ship_tracking/AIS_categories.csv | 282 ++++++++++++++++++++++++++ ship_tracking/anaconda-project.yml | 63 ++++++ ship_tracking/ship_tracking.ipynb | 310 +++++++++++++++++++++++++++++ 3 files changed, 655 insertions(+) create mode 100644 ship_tracking/AIS_categories.csv create mode 100644 ship_tracking/anaconda-project.yml create mode 100644 ship_tracking/ship_tracking.ipynb diff --git a/ship_tracking/AIS_categories.csv b/ship_tracking/AIS_categories.csv new file mode 100644 index 000000000..630270c4a --- /dev/null +++ b/ship_tracking/AIS_categories.csv @@ -0,0 +1,282 @@ +num,desc,category,category_desc +0,Not available,0,Unknown +1,Reserved,0,Unknown +2,Reserved,0,Unknown +3,Reserved,0,Unknown +4,Reserved,0,Unknown +5,Reserved,0,Unknown +6,Reserved,0,Unknown +7,Reserved,0,Unknown +8,Reserved,0,Unknown +9,Reserved,0,Unknown +10,Reserved,0,Unknown +11,Reserved,0,Unknown +12,Reserved,0,Unknown +13,Reserved,0,Unknown +14,Reserved,0,Unknown +15,Reserved,0,Unknown +16,Reserved,0,Unknown +17,Reserved,0,Unknown +18,Reserved,0,Unknown +19,Reserved,0,Unknown +20,"Wing in ground (WIG), all ships of this type",1,WIG +21,"Wing in ground (WIG), Hazardous category A",1,WIG +22,"Wing in ground (WIG), Hazardous category B",1,WIG +23,"Wing in ground (WIG), Hazardous category C",1,WIG +24,"Wing in ground (WIG), Hazardous category D",1,WIG +25,"Wing in ground (WIG), Reserved for future use",1,WIG +26,"Wing in ground (WIG), Reserved for future use",1,WIG +27,"Wing in ground (WIG), Reserved for future use",1,WIG +28,"Wing in ground (WIG), Reserved for future use",1,WIG +29,"Wing in ground (WIG), Reserved for future use",1,WIG +30,Fishing,2,Fishing +31,Towing,3,Towing +32,Towing: length exceeds 200m or breadth exceeds 25m,3,Towing +33,Dredging or underwater ops,4,Dredging +34,Diving ops,5,Diving +35,Military ops,6,Military +36,Sailing,7,Sailing +37,Pleasure Craft,8,Pleasure +38,Reserved,0,Unknown +39,Reserved,0,Unknown +40,"High speed craft (HSC), all ships of this type",9,High Speed +41,"High speed craft (HSC), Hazardous category A",9,High Speed +42,"High speed craft (HSC), Hazardous category B",9,High Speed +43,"High speed craft (HSC), Hazardous category C",9,High Speed +44,"High speed craft (HSC), Hazardous category D",9,High Speed +45,"High speed craft (HSC), Reserved for future use",9,High Speed +46,"High speed craft (HSC), Reserved for future use",9,High Speed +47,"High speed craft (HSC), Reserved for future use",9,High Speed +48,"High speed craft (HSC), Reserved for future use",9,High Speed +49,"High speed craft (HSC), No additional information",9,High Speed +50,Pilot Vessel,10,Pilot Vessel +51,Search and Rescue vessel,11,Search and Rescue vessel +52,Tug,12,Tug +53,Port Tender,18,Passenger +54,Anti-pollution equipment,13,Industrial +55,Law Enforcement,14,Law Enforcement +56,Spare - Local Vessel,15,Spare +57,Spare - Local Vessel,15,Spare +58,Medical Transport,16,Medical Transport +59,Noncombatant ship according to RR Resolution No. 18,17,Noncombatant +60,"Passenger, all ships of this type",18,Passenger +61,"Passenger, Hazardous category A",18,Passenger +62,"Passenger, Hazardous category B",18,Passenger +63,"Passenger, Hazardous category C",18,Passenger +64,"Passenger, Hazardous category D",18,Passenger +65,"Passenger, Reserved for future use",18,Passenger +66,"Passenger, Reserved for future use",18,Passenger +67,"Passenger, Reserved for future use",18,Passenger +68,"Passenger, Reserved for future use",18,Passenger +69,"Passenger, No additional information",18,Passenger +70,"Cargo, all ships of this type",19,Cargo +71,"Cargo, Hazardous category A",19,Cargo +72,"Cargo, Hazardous category B",19,Cargo +73,"Cargo, Hazardous category C",19,Cargo +74,"Cargo, Hazardous category D",19,Cargo +75,"Cargo, Reserved for future use",19,Cargo +76,"Cargo, Reserved for future use",19,Cargo +77,"Cargo, Reserved for future use",19,Cargo +78,"Cargo, Reserved for future use",19,Cargo +79,"Cargo, No additional information",19,Cargo +80,"Tanker, all ships of this type",20,Tanker +81,"Tanker, Hazardous category A",20,Tanker +82,"Tanker, Hazardous category B",20,Tanker +83,"Tanker, Hazardous category C",20,Tanker +84,"Tanker, Hazardous category D",20,Tanker +85,"Tanker, Reserved for future use",20,Tanker +86,"Tanker, Reserved for future use",20,Tanker +87,"Tanker, Reserved for future use",20,Tanker +88,"Tanker, Reserved for future use",20,Tanker +89,"Tanker, No additional information",20,Tanker +90,"Other Type, all ships of this type",21,Other +91,"Other Type, Hazardous category A",21,Other +92,"Other Type, Hazardous category B",21,Other +93,"Other Type, Hazardous category C",21,Other +94,"Other Type, Hazardous category D",21,Other +95,"Other Type, Reserved for future use",21,Other +96,"Other Type, Reserved for future use",21,Other +97,"Other Type, Reserved for future use",21,Other +98,"Other Type, Reserved for future use",21,Other +99,"Other Type, no additional information",21,Other +100,Reserved for future use,0,Unknown +101,Reserved for future use,0,Unknown +102,Reserved for future use,0,Unknown +103,Reserved for future use,0,Unknown +104,Reserved for future use,0,Unknown +105,Reserved for future use,0,Unknown +106,Reserved for future use,0,Unknown +107,Reserved for future use,0,Unknown +108,Reserved for future use,0,Unknown +109,Reserved for future use,0,Unknown +110,Reserved for future use,0,Unknown +111,Reserved for future use,0,Unknown +112,Reserved for future use,0,Unknown +113,Reserved for future use,0,Unknown +114,Reserved for future use,0,Unknown +115,Reserved for future use,0,Unknown +116,Reserved for future use,0,Unknown +117,Reserved for future use,0,Unknown +118,Reserved for future use,0,Unknown +119,Reserved for future use,0,Unknown +120,Reserved for future use,0,Unknown +121,Reserved for future use,0,Unknown +122,Reserved for future use,0,Unknown +123,Reserved for future use,0,Unknown +124,Reserved for future use,0,Unknown +125,Reserved for future use,0,Unknown +126,Reserved for future use,0,Unknown +127,Reserved for future use,0,Unknown +128,Reserved for future use,0,Unknown +129,Reserved for future use,0,Unknown +130,Reserved for future use,0,Unknown +131,Reserved for future use,0,Unknown +132,Reserved for future use,0,Unknown +133,Reserved for future use,0,Unknown +134,Reserved for future use,0,Unknown +135,Reserved for future use,0,Unknown +136,Reserved for future use,0,Unknown +137,Reserved for future use,0,Unknown +138,Reserved for future use,0,Unknown +139,Reserved for future use,0,Unknown +140,Reserved for future use,0,Unknown +141,Reserved for future use,0,Unknown +142,Reserved for future use,0,Unknown +143,Reserved for future use,0,Unknown +144,Reserved for future use,0,Unknown +145,Reserved for future use,0,Unknown +146,Reserved for future use,0,Unknown +147,Reserved for future use,0,Unknown +148,Reserved for future use,0,Unknown +149,Reserved for future use,0,Unknown +150,Reserved for future use,0,Unknown +151,Reserved for future use,0,Unknown +152,Reserved for future use,0,Unknown +153,Reserved for future use,0,Unknown +154,Reserved for future use,0,Unknown +155,Reserved for future use,0,Unknown +156,Reserved for future use,0,Unknown +157,Reserved for future use,0,Unknown +158,Reserved for future use,0,Unknown +159,Reserved for future use,0,Unknown +160,Reserved for future use,0,Unknown +161,Reserved for future use,0,Unknown +162,Reserved for future use,0,Unknown +163,Reserved for future use,0,Unknown +164,Reserved for future use,0,Unknown +165,Reserved for future use,0,Unknown +166,Reserved for future use,0,Unknown +167,Reserved for future use,0,Unknown +168,Reserved for future use,0,Unknown +169,Reserved for future use,0,Unknown +170,Reserved for future use,0,Unknown +171,Reserved for future use,0,Unknown +172,Reserved for future use,0,Unknown +173,Reserved for future use,0,Unknown +174,Reserved for future use,0,Unknown +175,Reserved for future use,0,Unknown +176,Reserved for future use,0,Unknown +177,Reserved for future use,0,Unknown +178,Reserved for future use,0,Unknown +179,Reserved for future use,0,Unknown +180,Reserved for future use,0,Unknown +181,Reserved for future use,0,Unknown +182,Reserved for future use,0,Unknown +183,Reserved for future use,0,Unknown +184,Reserved for future use,0,Unknown +185,Reserved for future use,0,Unknown +186,Reserved for future use,0,Unknown +187,Reserved for future use,0,Unknown +188,Reserved for future use,0,Unknown +189,Reserved for future use,0,Unknown +190,Reserved for future use,0,Unknown +191,Reserved for future use,0,Unknown +192,Reserved for future use,0,Unknown +193,Reserved for future use,0,Unknown +194,Reserved for future use,0,Unknown +195,Reserved for future use,0,Unknown +196,Reserved for future use,0,Unknown +197,Reserved for future use,0,Unknown +198,Reserved for future use,0,Unknown +199,Reserved for future use,0,Unknown +200,Reserved for future use,0,Unknown +201,Reserved for future use,0,Unknown +202,Reserved for future use,0,Unknown +203,Reserved for future use,0,Unknown +204,Reserved for future use,0,Unknown +205,Reserved for future use,0,Unknown +206,Reserved for future use,0,Unknown +207,Reserved for future use,0,Unknown +208,Reserved for future use,0,Unknown +209,Reserved for future use,0,Unknown +210,Reserved for future use,0,Unknown +211,Reserved for future use,0,Unknown +212,Reserved for future use,0,Unknown +213,Reserved for future use,0,Unknown +214,Reserved for future use,0,Unknown +215,Reserved for future use,0,Unknown +216,Reserved for future use,0,Unknown +217,Reserved for future use,0,Unknown +218,Reserved for future use,0,Unknown +219,Reserved for future use,0,Unknown +220,Reserved for future use,0,Unknown +221,Reserved for future use,0,Unknown +222,Reserved for future use,0,Unknown +223,Reserved for future use,0,Unknown +224,Reserved for future use,0,Unknown +225,Reserved for future use,0,Unknown +226,Reserved for future use,0,Unknown +227,Reserved for future use,0,Unknown +228,Reserved for future use,0,Unknown +229,Reserved for future use,0,Unknown +230,Reserved for future use,0,Unknown +231,Reserved for future use,0,Unknown +232,Reserved for future use,0,Unknown +233,Reserved for future use,0,Unknown +234,Reserved for future use,0,Unknown +235,Reserved for future use,0,Unknown +236,Reserved for future use,0,Unknown +237,Reserved for future use,0,Unknown +238,Reserved for future use,0,Unknown +239,Reserved for future use,0,Unknown +240,Reserved for future use,0,Unknown +241,Reserved for future use,0,Unknown +242,Reserved for future use,0,Unknown +243,Reserved for future use,0,Unknown +244,Reserved for future use,0,Unknown +245,Reserved for future use,0,Unknown +246,Reserved for future use,0,Unknown +247,Reserved for future use,0,Unknown +248,Reserved for future use,0,Unknown +249,Reserved for future use,0,Unknown +250,Reserved for future use,0,Unknown +251,Reserved for future use,0,Unknown +252,Reserved for future use,0,Unknown +253,Reserved for future use,0,Unknown +254,Reserved for future use,0,Unknown +255,Reserved for future use,0,Unknown +1001,Fishing vessels,2,Fishing +1002,Fishing vessels,2,Fishing +1003,Freight Vessels,19,Cargo +1004,Freight Vessels,19,Cargo +1005,Industrial vessels,13,Industrial +1006,Miscellaneous vessels,21,Other +1007,Offshore drilling vessels,13,Industrial +1008,non-vessel,21,Other +1009,non-vessel,21,Other +1010,Offshore supply vessel,13,Industrial +1011,Oil Recovery vessel,13,Industrial +1012,Passenger ships,18,Passenger +1013,Passenger ships,18,Passenger +1014,Passenger ships,18,Passenger +1015,Passenger ships,18,Passenger +1016,Public freight,19,Cargo +1017,Public tankship/barge,20,Tanker +1018,Unclassified public vessel,0,Unknown +1019,Recreational Vessel,8,Pleasure +1020,Research Vessel,21,Other +1021,SAR Aircraft,21,Other +1022,School ship,21,Other +1023,Tank Barge,20,Tanker +1024,Tank Ship,20,Tanker +1025,Towing Vessel,3,Towing diff --git a/ship_tracking/anaconda-project.yml b/ship_tracking/anaconda-project.yml new file mode 100644 index 000000000..b9c2ba198 --- /dev/null +++ b/ship_tracking/anaconda-project.yml @@ -0,0 +1,63 @@ +# To reproduce: install 'anaconda-project', then 'anaconda-project run' +name: ship_tracking +description: Visualizing AIS tracking data for ships near the USA +maintainers: +- jbednar +labels: +- datashader +- holoviews + +user_fields: [labels, skip, maintainers] + +channels: + - pyviz + +packages: &pkgs + - bokeh ==2.2.3 + - colorcet ==2 + - dask ==2020.12.0 + - datashader ==0.12.0 + - holoviews ==1.14.0 + - notebook ==6.1.5 + - numba ==0.51.2 + - numexpr ==2.7.1 + - pandas ==1.1.5 + - panel ==0.10.3 + - python ==3.7.9 + - spatialpandas ==0.3.6 + - xarray ==0.16.2 + - pip ==20.3.3 + +dependencies: *pkgs + +commands: + dashboard: + unix: panel serve ship_tracking.ipynb + supports_http_options: true + notebook: + notebook: ship_tracking.ipynb + test: + unix: pytest --nbsmoke-run -k *.ipynb --ignore envs + windows: pytest --nbsmoke-run -k *.ipynb --ignore envs + env_spec: test + lint: + unix: pytest --nbsmoke-lint -k *.ipynb --ignore envs + windows: pytest --nbsmoke-lint -k *.ipynb --ignore envs + env_spec: test + +variables: {} +downloads: + DATAFILE: + url: https://zenodo.org/record/3541812/files/HG_OOSTENDE-gps-2018.csv + filename: data/HG_OOSTENDE-gps-2018.csv + +env_specs: + default: {} + test: + packages: + - nbsmoke=0.2.8 + - pytest=4.4.1 +platforms: +- linux-64 +- osx-64 +- win-64 diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb new file mode 100644 index 000000000..57cc54d1f --- /dev/null +++ b/ship_tracking/ship_tracking.ipynb @@ -0,0 +1,310 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exploring AIS vessel-tracking data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os, numpy as np, pandas as pd, panel as pn, colorcet as cc, datashader as ds, holoviews as hv\n", + "import spatialpandas as sp, spatialpandas.io, spatialpandas.geometry, spatialpandas.dask, dask.dataframe as dd\n", + "\n", + "from glob import glob\n", + "from holoviews.util.transform import lon_lat_to_easting_northing as ll2en\n", + "from holoviews.operation.datashader import rasterize, datashade, dynspread\n", + "from IPython.display import display, HTML\n", + "from dask.diagnostics import ProgressBar\n", + "\n", + "display(HTML(\"\"))\n", + "hv.extension('bokeh')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Vessel categories \n", + "\n", + "AIS pings come with an associated integer `VesselType`, which broadly labels what sort of vessel it is. Different types of vessels are used for different purposes and behave differently, as we can see if we color-code the location of each ping by the `VesselType` using Datshader. \n", + "\n", + "Type names are defined in a separate file constructed using lists of 100+ [AIS Vessel Types](https://api.vtexplorer.com/docs/ref-aistypes.html), and can be further collapsed into a smaller number of broad vessel categories:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vessel_types=pd.read_csv(\"AIS_categories.csv\")\n", + "vessel_types.head(37).tail(3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can further reduce the `category` to the 6 most common (with the rest as `Other`). We will create a dictionary which maps the value to one of the categories:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "categories = {r.num: r.category if r.category in [0,2,3,19,12,18] else 21 for i, r in vessel_types.iterrows()}\n", + "categories[np.NaN] = 0\n", + "\n", + "def category_desc(val):\n", + " \"\"\"Return description for the category with the indicated integer value\"\"\"\n", + " return vessel_types[vessel_types.category==val].iloc[0].category_desc\n", + "\n", + "vessel_mapping = dict(zip(vessel_types.num.to_list(), vessel_types.category.to_list()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let us look at the categories:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "groups = {categories[i]: category_desc(categories[i]) for i in vessel_types.num.unique()}\n", + "print(\" \".join([f\"{k}: {v}\" for k,v in sorted(groups.items())]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Given a set of colors, let's construct a color key for Datashader to use later, along with a visible legend we can add to such a plot:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "colors = cc.glasbey_bw_minc_20_minl_30\n", + "color_key = {list(groups.keys())[i]:tuple(int(e*255.) for e in v) for i,v in \n", + " enumerate(colors[:(len(groups))][::-1])}\n", + "legend = hv.NdOverlay({groups[k]: hv.Points([0,0], label=str(groups[k])).opts(\n", + " color=cc.rgb_to_hex(*v), size=0) \n", + " for k, v in color_key.items()})\n", + "#legend #uncomment to see legend alone" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load AIS pings\n", + "\n", + "Next we will load the data from disk, either directly from a spatially indexed Parquet file (if previously cached) or from the raw CSV files. We'll also project the data to the coordinate system we will use later for plotting.\n", + "\n", + "Since particularly in raw form this is a lot of data, we will use the `map_partitions` functionality of a dask.DataFrame. To do this we define a function to the conversion and an example DataFrame with the required structure:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def convert_partition(df):\n", + " east, north = ll2en(df.LON.astype('float32'), df.LAT.astype('float32'))\n", + " return sp.GeoDataFrame({\n", + " 'geometry': sp.geometry.PointArray((east, north)),\n", + " 'MMSI': df.MMSI.fillna(0).astype('int32'),\n", + " 'category': df.VesselType.replace(categories).astype('int32')}\n", + " )\n", + "\n", + "example = sp.GeoDataFrame({\n", + " 'geometry': sp.geometry.PointArray([], dtype='int32'),\n", + " 'MMSI': np.array([], dtype='int32'),\n", + " 'category': np.array([], dtype='int32')\n", + "})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we will define the function that will load our data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "basedir = './2020/'\n", + "basename = 'AIS_2020_01'\n", + "\n", + "def load_data():\n", + " # Read much-smaller cached Parquet file from disk if available\n", + " cache_file = basedir+basename+\"_gdf.parquet\"\n", + " if os.path.exists(cache_file):\n", + " print(\"Reading Parquet file\")\n", + " return sp.io.read_parquet_dask(cache_file).persist()\n", + "\n", + " df = dd.read_csv(basedir+'AIS_2020_01*.csv')\n", + " with ProgressBar():\n", + " print(\"Reading CSVs\")\n", + " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", + " print(\"Writing Parquet file\")\n", + " return gdf.pack_partitions_to_parquet(cache_file, npartitions=16).persist()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Actually load the data, using the disk cache and memory cache if available:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "df = pn.state.as_cached('df', load_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Datashaded, categorical AIS plot (Zone 10)\n", + "\n", + "We can now plot the data colored by category, with a color key.\n", + "\n", + "To zoom in & interact with the plot, click the “Wheel zoom” tool in the toolbar on the side of the plot.\n", + "Click-and-drag the plot in order to look around. As you zoom in, finer-grained detail will emerge and fill in. Depending on the size of the dataset and your machine, this might take a second." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "x_range, y_range = ll2en([-126,-120.7], [47.5,49.5])\n", + "bounds = dict(x=tuple(x_range), y=tuple(y_range))\n", + "\n", + "pts = hv.Points(df, vdims=['category']).redim.range(**bounds)\n", + "points = dynspread(datashade(pts, aggregator=ds.count_cat('category'), color_key=color_key))\n", + "\n", + "tiles = hv.element.tiles.ESRI().opts(alpha=0.4, bgcolor=\"black\").opts(responsive=True, min_height=600)\n", + "labels = hv.element.tiles.StamenLabels().opts(alpha=0.7, level='glyph')\n", + "\n", + "tiles * points * labels * legend" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clearly, the ship's behavior is highly dependent on category, with very different patterns of motion between these categories (and presumably the other categories not shown). E.g. passenger vessels tend to travel _across_ narrow waterways, while towing and cargo vessels travel _along_ them. Fishing vessels, as one would expect, travel out to open water and then cover a wide area around their initial destination. Zooming and panning (using the [Bokeh](https://docs.bokeh.org/en/latest/docs/user_guide/tools.html) tools at the right) reveal other patterns at different locations and scales.\n", + "\n", + "# Selecting specific datapoints\n", + "\n", + "To help understand clusters of datapoints or individual datapoints, we can use the x,y location of a tap to query the dataset for pings in that region, then highlight them on top of the main plot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def highlight(x,y, delta = 0.02, max_points=1):\n", + " pts = []\n", + " if None not in [x,y]:\n", + " selection = df.cx[x-delta:x+delta, y-delta:y+delta]\n", + " if len(selection) > 0:\n", + " pts = list(selection[:max_points])\n", + " return hv.Points(pts).opts(color='white', marker='o', size=8)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pointsp = dynspread(datashade(pts, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", + "selected = hv.DynamicMap(highlight, streams=[hv.streams.Tap()])\n", + "\n", + "#tiles * pointsp * selected * legend" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We could view the result above by uncommenting the last line, but let's just go ahead and make a little app so that we can let the user decide whether to have labels visible:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def labels(enable=True):\n", + " return hv.element.tiles.StamenLabels().opts(level='glyph', alpha=0.9 if enable else 0)\n", + "\n", + "show_labels = pn.widgets.Checkbox(name=\"Show labels\", value=True)\n", + "overlay = tiles * pointsp * selected * hv.DynamicMap(pn.bind(labels, enable=show_labels)) * legend\n", + " \n", + "pn.Column(\"# Categorical plot of AIS data by type\",\n", + " \"Zoom or pan to explore the data, then click to select \"\n", + " \"and highlight connected vessel tracks in a region. \",\n", + " \"You may need to zoom in before a track is selectable.\",\n", + " show_labels, overlay).servable()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From a2b994e5418dc21bc6472cc883580af59a71e331 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Mon, 18 Jan 2021 20:12:04 -0600 Subject: [PATCH 02/46] Formatting --- ship_tracking/ship_tracking.ipynb | 37 +++++++++++-------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 57cc54d1f..e9efd11f0 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -131,21 +131,19 @@ " return sp.GeoDataFrame({\n", " 'geometry': sp.geometry.PointArray((east, north)),\n", " 'MMSI': df.MMSI.fillna(0).astype('int32'),\n", - " 'category': df.VesselType.replace(categories).astype('int32')}\n", - " )\n", + " 'category': df.VesselType.replace(categories).astype('int32')})\n", "\n", "example = sp.GeoDataFrame({\n", " 'geometry': sp.geometry.PointArray([], dtype='int32'),\n", - " 'MMSI': np.array([], dtype='int32'),\n", - " 'category': np.array([], dtype='int32')\n", - "})" + " 'MMSI': np.array([], dtype='int32'),\n", + " 'category': np.array([], dtype='int32')})" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Next we will define the function that will load our data:" + "Next we will define the function that will load our data, reading a much-smaller (and much faster to load) cached Parquet-format file from disk if available:" ] }, { @@ -158,7 +156,6 @@ "basename = 'AIS_2020_01'\n", "\n", "def load_data():\n", - " # Read much-smaller cached Parquet file from disk if available\n", " cache_file = basedir+basename+\"_gdf.parquet\"\n", " if os.path.exists(cache_file):\n", " print(\"Reading Parquet file\")\n", @@ -204,9 +201,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "x_range, y_range = ll2en([-126,-120.7], [47.5,49.5])\n", @@ -284,25 +279,19 @@ " \"You may need to zoom in before a track is selectable.\",\n", " show_labels, overlay).servable()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.10" + "pygments_lexer": "ipython3" } }, "nbformat": 4, From 4e60d0fae772b89881ee6eeae5b5fd9d12c4dd5f Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Tue, 19 Jan 2021 10:04:04 -0600 Subject: [PATCH 03/46] Added usecols to speed up CSV reading Co-authored-by: Philipp Rudiger --- ship_tracking/ship_tracking.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index e9efd11f0..09f60772a 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -161,7 +161,8 @@ " print(\"Reading Parquet file\")\n", " return sp.io.read_parquet_dask(cache_file).persist()\n", "\n", - " df = dd.read_csv(basedir+'AIS_2020_01*.csv')\n", + " df = dd.read_csv(basedir+basename+'*.csv', usecols=['MMSI', 'LON', 'LAT', 'VesselType']) +\n", " with ProgressBar():\n", " print(\"Reading CSVs\")\n", " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", From 6517ff4c2f9cc436a943d212856b6c14ca6fd02d Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Tue, 19 Jan 2021 10:04:47 -0600 Subject: [PATCH 04/46] Doubly persisting speeds up rendering 3-10x Co-authored-by: Philipp Rudiger --- ship_tracking/ship_tracking.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 09f60772a..741a72f8b 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -159,7 +159,7 @@ " cache_file = basedir+basename+\"_gdf.parquet\"\n", " if os.path.exists(cache_file):\n", " print(\"Reading Parquet file\")\n", - " return sp.io.read_parquet_dask(cache_file).persist()\n", + " return sp.io.read_parquet_dask(cache_file).persist().persist()\n", "\n", " df = dd.read_csv(basedir+basename+'*.csv', usecols=['MMSI', 'LON', 'LAT', 'VesselType']) \n", From c44c2609d61c0f86b2b49511a3db297d83315bbd Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Tue, 19 Jan 2021 10:05:53 -0600 Subject: [PATCH 05/46] Allow larger npartitions --- ship_tracking/ship_tracking.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 741a72f8b..1ab64e2f0 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -167,7 +167,7 @@ " print(\"Reading CSVs\")\n", " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", " print(\"Writing Parquet file\")\n", - " return gdf.pack_partitions_to_parquet(cache_file, npartitions=16).persist()" + " return gdf.pack_partitions_to_parquet(cache_file).persist()" ] }, { From 95c9d76ef31e280586ae7a6f8229827b2c27d531 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Tue, 19 Jan 2021 17:04:01 +0100 Subject: [PATCH 06/46] Update loading code --- ship_tracking/ship_tracking.ipynb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 1ab64e2f0..b5173e5d6 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -161,13 +161,12 @@ " print(\"Reading Parquet file\")\n", " return sp.io.read_parquet_dask(cache_file).persist().persist()\n", "\n", - " df = dd.read_csv(basedir+basename+'*.csv', usecols=['MMSI', 'LON', 'LAT', 'VesselType']) -\n", + " df = dd.read_csv(basedir+basename+'*.csv', usecols=['MMSI', 'LON', 'LAT', 'VesselType'])\n", " with ProgressBar():\n", - " print(\"Reading CSVs\")\n", + " print('Reading CSV files')\n", " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", " print(\"Writing Parquet file\")\n", - " return gdf.pack_partitions_to_parquet(cache_file).persist()" + " return gdf.pack_partitions_to_parquet(cache_file).persist().persist()" ] }, { From 1e7f1c13eaa9b40708e95da0beccf6952e78997f Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Tue, 19 Jan 2021 10:21:14 -0600 Subject: [PATCH 07/46] Fixed example datatype --- ship_tracking/ship_tracking.ipynb | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index b5173e5d6..692bdb069 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -130,11 +130,11 @@ " east, north = ll2en(df.LON.astype('float32'), df.LAT.astype('float32'))\n", " return sp.GeoDataFrame({\n", " 'geometry': sp.geometry.PointArray((east, north)),\n", - " 'MMSI': df.MMSI.fillna(0).astype('int32'),\n", + " 'MMSI': df.MMSI.fillna(0).astype('int32'),\n", " 'category': df.VesselType.replace(categories).astype('int32')})\n", "\n", "example = sp.GeoDataFrame({\n", - " 'geometry': sp.geometry.PointArray([], dtype='int32'),\n", + " 'geometry': sp.geometry.PointArray([], dtype='float32'),\n", " 'MMSI': np.array([], dtype='int32'),\n", " 'category': np.array([], dtype='int32')})" ] @@ -156,16 +156,16 @@ "basename = 'AIS_2020_01'\n", "\n", "def load_data():\n", - " cache_file = basedir+basename+\"_gdf.parquet\"\n", + " cache_file = basedir+basename+'_gdf.parquet'\n", " if os.path.exists(cache_file):\n", - " print(\"Reading Parquet file\")\n", + " print('Reading Parquet file')\n", " return sp.io.read_parquet_dask(cache_file).persist().persist()\n", "\n", " df = dd.read_csv(basedir+basename+'*.csv', usecols=['MMSI', 'LON', 'LAT', 'VesselType'])\n", " with ProgressBar():\n", " print('Reading CSV files')\n", " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", - " print(\"Writing Parquet file\")\n", + " print('Writing Parquet file')\n", " return gdf.pack_partitions_to_parquet(cache_file).persist().persist()" ] }, @@ -279,13 +279,6 @@ " \"You may need to zoom in before a track is selectable.\",\n", " show_labels, overlay).servable()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 4e216515f328c72aaef555312abd00214050b9d3 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Tue, 19 Jan 2021 10:21:39 -0600 Subject: [PATCH 08/46] Force pyarrow version 2 for orders of magnitude speedup --- ship_tracking/anaconda-project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ship_tracking/anaconda-project.yml b/ship_tracking/anaconda-project.yml index b9c2ba198..972c7cdca 100644 --- a/ship_tracking/anaconda-project.yml +++ b/ship_tracking/anaconda-project.yml @@ -27,6 +27,7 @@ packages: &pkgs - spatialpandas ==0.3.6 - xarray ==0.16.2 - pip ==20.3.3 + - conda-forge::pyarrow ==2 dependencies: *pkgs From 18da3757da07ae3461ac8c0b4938edee3ac3ae10 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Tue, 19 Jan 2021 17:46:49 +0100 Subject: [PATCH 09/46] Persist categories --- ship_tracking/ship_tracking.ipynb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 692bdb069..4f4ee5285 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -158,15 +158,19 @@ "def load_data():\n", " cache_file = basedir+basename+'_gdf.parquet'\n", " if os.path.exists(cache_file):\n", - " print('Reading Parquet file')\n", - " return sp.io.read_parquet_dask(cache_file).persist().persist()\n", + " print(\"Reading Parquet file\")\n", + " gdf = sp.io.read_parquet_dask(cache_file)\n", + " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", + " return gdf.persist()\n", "\n", " df = dd.read_csv(basedir+basename+'*.csv', usecols=['MMSI', 'LON', 'LAT', 'VesselType'])\n", " with ProgressBar():\n", " print('Reading CSV files')\n", " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", - " print('Writing Parquet file')\n", - " return gdf.pack_partitions_to_parquet(cache_file).persist().persist()" + " print(\"Writing Parquet file\")\n", + " gdf = gdf.pack_partitions_to_parquet(cache_file, npartitions=64)\n", + " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", + " return gdf.persist()" ] }, { From 4b845c19ae9569ad64da599d7abd72302833a8bf Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Tue, 19 Jan 2021 10:50:08 -0600 Subject: [PATCH 10/46] Pull out usecols --- ship_tracking/ship_tracking.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 4f4ee5285..8c9d32c17 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -154,6 +154,7 @@ "source": [ "basedir = './2020/'\n", "basename = 'AIS_2020_01'\n", + "usecols = ['MMSI', 'LON', 'LAT', 'VesselType']\n", "\n", "def load_data():\n", " cache_file = basedir+basename+'_gdf.parquet'\n", @@ -163,7 +164,7 @@ " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", " return gdf.persist()\n", "\n", - " df = dd.read_csv(basedir+basename+'*.csv', usecols=['MMSI', 'LON', 'LAT', 'VesselType'])\n", + " df = dd.read_csv(basedir+basename+'*.csv', usecols=usecols)\n", " with ProgressBar():\n", " print('Reading CSV files')\n", " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", From 2871d4a0f44636a60475af691d0f187a03db75ac Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Tue, 19 Jan 2021 18:17:31 -0600 Subject: [PATCH 11/46] Now collects vessel info (for hover/drilldown) alongside pings --- ship_tracking/ship_tracking.ipynb | 47 ++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 8c9d32c17..5a2f191d5 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -44,7 +44,7 @@ "outputs": [], "source": [ "vessel_types=pd.read_csv(\"AIS_categories.csv\")\n", - "vessel_types.head(37).tail(3)" + "vessel_types.iloc[34:37]" ] }, { @@ -154,24 +154,39 @@ "source": [ "basedir = './2020/'\n", "basename = 'AIS_2020_01'\n", - "usecols = ['MMSI', 'LON', 'LAT', 'VesselType']\n", + "index = 'MMSI'\n", + "dfcols = ['MMSI', 'LON', 'LAT', 'BaseDateTime', 'VesselType']\n", + "vesselcols = ['MMSI', 'IMO', 'CallSign', 'VesselName', 'VesselType', 'Length', 'Width']\n", "\n", "def load_data():\n", - " cache_file = basedir+basename+'_gdf.parquet'\n", - " if os.path.exists(cache_file):\n", - " print(\"Reading Parquet file\")\n", + " cache_file = basedir+basename+'_broadcast.parq'\n", + " vessels_file = basedir+basename+'_vessels.parq'\n", + " \n", + " if (os.path.exists(cache_file) and \n", + " os.path.exists(vessels_file)):\n", + " print('Reading parquet file')\n", " gdf = sp.io.read_parquet_dask(cache_file)\n", " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", - " return gdf.persist()\n", + " \n", + " print('Reading vessel info file')\n", + " vessels = dd.read_parquet(vessels_file)\n", "\n", - " df = dd.read_csv(basedir+basename+'*.csv', usecols=usecols)\n", - " with ProgressBar():\n", - " print('Reading CSV files')\n", - " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", - " print(\"Writing Parquet file\")\n", - " gdf = gdf.pack_partitions_to_parquet(cache_file, npartitions=64)\n", - " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", - " return gdf.persist()" + " else:\n", + " df = dd.read_csv(basedir+basename+'*.csv', usecols=dfcols+vesselcols, assume_missing=True)\n", + " with ProgressBar():\n", + " print('Reading CSV files')\n", + " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", + "\n", + " print('Writing vessel info file')\n", + " vessels = df[vesselcols].sample(frac=1).drop_duplicates([index]).compute()\n", + " vessels[index] = vessels[index].astype('int32')\n", + " vessels.to_parquet(vessels_file)\n", + "\n", + " print('Writing parquet file')\n", + " gdf = gdf.pack_partitions_to_parquet(cache_file, npartitions=64)\n", + " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", + " \n", + " return gdf.persist(), vessels" ] }, { @@ -188,7 +203,7 @@ "outputs": [], "source": [ "%%time\n", - "df = pn.state.as_cached('df', load_data)" + "df, vessels = pn.state.as_cached('df', load_data)" ] }, { @@ -282,7 +297,7 @@ " \"Zoom or pan to explore the data, then click to select \"\n", " \"and highlight connected vessel tracks in a region. \",\n", " \"You may need to zoom in before a track is selectable.\",\n", - " show_labels, overlay).servable()" + " show_labels, overlay, sizing_mode='stretch_width').servable()" ] } ], From bee294cf827b8a73d48c2b2cae0a6bb0100eb677 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Wed, 20 Jan 2021 14:45:42 +0000 Subject: [PATCH 12/46] Removed unnecessary HTML at the top of the notebook --- ship_tracking/ship_tracking.ipynb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 5a2f191d5..f33d877c3 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -22,8 +22,7 @@ "from IPython.display import display, HTML\n", "from dask.diagnostics import ProgressBar\n", "\n", - "display(HTML(\"\"))\n", - "hv.extension('bokeh')" + "hv.extension('bokeh', width=100)" ] }, { From 7eb460c4934b162e75d0b9b1e098d9edfce63b14 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Wed, 20 Jan 2021 14:47:26 +0000 Subject: [PATCH 13/46] Removed unused imports --- ship_tracking/ship_tracking.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index f33d877c3..d6fa98a5d 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -19,7 +19,6 @@ "from glob import glob\n", "from holoviews.util.transform import lon_lat_to_easting_northing as ll2en\n", "from holoviews.operation.datashader import rasterize, datashade, dynspread\n", - "from IPython.display import display, HTML\n", "from dask.diagnostics import ProgressBar\n", "\n", "hv.extension('bokeh', width=100)" From 136c3588a9fe6fb3f9293dd17aa1e9d2c9d7941d Mon Sep 17 00:00:00 2001 From: jlstevens Date: Wed, 20 Jan 2021 21:24:59 +0000 Subject: [PATCH 14/46] Added inspect_points operation to notebook overlay --- ship_tracking/ship_tracking.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index d6fa98a5d..600537425 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -18,7 +18,7 @@ "\n", "from glob import glob\n", "from holoviews.util.transform import lon_lat_to_easting_northing as ll2en\n", - "from holoviews.operation.datashader import rasterize, datashade, dynspread\n", + "from holoviews.operation.datashader import rasterize, datashade, dynspread, inspect_points\n", "from dask.diagnostics import ProgressBar\n", "\n", "hv.extension('bokeh', width=100)" @@ -231,7 +231,9 @@ "tiles = hv.element.tiles.ESRI().opts(alpha=0.4, bgcolor=\"black\").opts(responsive=True, min_height=600)\n", "labels = hv.element.tiles.StamenLabels().opts(alpha=0.7, level='glyph')\n", "\n", - "tiles * points * labels * legend" + "inspect = inspect_points(points, streams=[hv.streams.Tap], link_inputs=False).opts(color='red', \n", + " tools=['hover'], marker='square', size=10, fill_alpha=0)\n", + "tiles * points * labels * legend * inspect" ] }, { From 3bc853f0f9e40491899246874177c75e1bac428d Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 20 Jan 2021 16:58:03 -0600 Subject: [PATCH 15/46] Updated text. Moved inspection/highlight to the final app --- ship_tracking/ship_tracking.ipynb | 60 +++++++++++++------------------ 1 file changed, 24 insertions(+), 36 deletions(-) diff --git a/ship_tracking/ship_tracking.ipynb b/ship_tracking/ship_tracking.ipynb index 600537425..95a09b960 100644 --- a/ship_tracking/ship_tracking.ipynb +++ b/ship_tracking/ship_tracking.ipynb @@ -4,7 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Exploring AIS vessel-tracking data" + "# Exploring AIS vessel-traffic data\n", + "\n", + "This [Jupyter](https://jupyter.org) notebook demonstrates how to use the [Datashader](https://datashader.org)-based rendering in [HoloViews](https://holoviews.org) to explore and analyze US Coast Guard [Automatic Identification System (AIS)](https://en.wikipedia.org/wiki/Automatic_identification_system) vessel-location data. Vessels are identified by their [Maritime Mobile Service Identity](https://en.wikipedia.org/wiki/Maritime_Mobile_Service_Identity) numbers, and other data about the vessels is also typically included. Data is provided for January 2020, but additional months and years of data can be downloaded for US coastal areas from [marinecadastre.gov](marinehttps://marinecadastre.gov/ais), and similar approaches should be usable for other AIS data available for other regions." ] }, { @@ -82,7 +84,7 @@ "outputs": [], "source": [ "groups = {categories[i]: category_desc(categories[i]) for i in vessel_types.num.unique()}\n", - "print(\" \".join([f\"{k}: {v}\" for k,v in sorted(groups.items())]))" + "print(\" \".join([f\"{k}:{v}\" for k,v in sorted(groups.items())]))" ] }, { @@ -160,15 +162,14 @@ " cache_file = basedir+basename+'_broadcast.parq'\n", " vessels_file = basedir+basename+'_vessels.parq'\n", " \n", - " if (os.path.exists(cache_file) and \n", - " os.path.exists(vessels_file)):\n", + " if (os.path.exists(cache_file) and os.path.exists(vessels_file)):\n", + " print('Reading vessel info file')\n", + " vessels = dd.read_parquet(vessels_file)\n", + "\n", " print('Reading parquet file')\n", " gdf = sp.io.read_parquet_dask(cache_file)\n", " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", " \n", - " print('Reading vessel info file')\n", - " vessels = dd.read_parquet(vessels_file)\n", - "\n", " else:\n", " df = dd.read_csv(basedir+basename+'*.csv', usecols=dfcols+vesselcols, assume_missing=True)\n", " with ProgressBar():\n", @@ -208,12 +209,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Datashaded, categorical AIS plot (Zone 10)\n", + "# Plot categorical data\n", "\n", "We can now plot the data colored by category, with a color key.\n", "\n", - "To zoom in & interact with the plot, click the “Wheel zoom” tool in the toolbar on the side of the plot.\n", - "Click-and-drag the plot in order to look around. As you zoom in, finer-grained detail will emerge and fill in. Depending on the size of the dataset and your machine, this might take a second." + "To zoom in & interact with the plot, click the “Wheel zoom” tool in the toolbar on the side of the plot. Click and drag the plot in order to look around. As you zoom in, finer-grained detail will emerge and fill in, as long as you have a live Python process running to render the data dynamically. Depending on the size of the dataset and your machine, updating the plot might take a few seconds." ] }, { @@ -222,7 +222,7 @@ "metadata": {}, "outputs": [], "source": [ - "x_range, y_range = ll2en([-126,-120.7], [47.5,49.5])\n", + "x_range, y_range = ll2en([-54,-128], [15,56])\n", "bounds = dict(x=tuple(x_range), y=tuple(y_range))\n", "\n", "pts = hv.Points(df, vdims=['category']).redim.range(**bounds)\n", @@ -231,9 +231,7 @@ "tiles = hv.element.tiles.ESRI().opts(alpha=0.4, bgcolor=\"black\").opts(responsive=True, min_height=600)\n", "labels = hv.element.tiles.StamenLabels().opts(alpha=0.7, level='glyph')\n", "\n", - "inspect = inspect_points(points, streams=[hv.streams.Tap], link_inputs=False).opts(color='red', \n", - " tools=['hover'], marker='square', size=10, fill_alpha=0)\n", - "tiles * points * labels * legend * inspect" + "tiles * points * labels * legend" ] }, { @@ -244,22 +242,7 @@ "\n", "# Selecting specific datapoints\n", "\n", - "To help understand clusters of datapoints or individual datapoints, we can use the x,y location of a tap to query the dataset for pings in that region, then highlight them on top of the main plot." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def highlight(x,y, delta = 0.02, max_points=1):\n", - " pts = []\n", - " if None not in [x,y]:\n", - " selection = df.cx[x-delta:x+delta, y-delta:y+delta]\n", - " if len(selection) > 0:\n", - " pts = list(selection[:max_points])\n", - " return hv.Points(pts).opts(color='white', marker='o', size=8)" + "To help understand clusters of datapoints or individual datapoints, we can use the x,y location of a tap to query the dataset for a ping in that region, then highlight it on top of the main plot." ] }, { @@ -268,10 +251,14 @@ "metadata": {}, "outputs": [], "source": [ - "pointsp = dynspread(datashade(pts, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", - "selected = hv.DynamicMap(highlight, streams=[hv.streams.Tap()])\n", + "xr, yr = ll2en([-126,-120.7], [47.5,49.5])\n", + "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", + "pointsp = dynspread(datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", + "\n", + "highlight = inspect_points(pointsp, streams=[hv.streams.Tap], link_inputs=False)\n", + "highlight = highlight.opts(color='white', tools=[\"hover\"], marker='square', size=10, fill_alpha=0)\n", "\n", - "#tiles * pointsp * selected * legend" + "#tiles * pointsp * highlight * legend" ] }, { @@ -287,11 +274,12 @@ "metadata": {}, "outputs": [], "source": [ - "def labels(enable=True):\n", + "def label_fn(enable=True):\n", " return hv.element.tiles.StamenLabels().opts(level='glyph', alpha=0.9 if enable else 0)\n", - "\n", "show_labels = pn.widgets.Checkbox(name=\"Show labels\", value=True)\n", - "overlay = tiles * pointsp * selected * hv.DynamicMap(pn.bind(labels, enable=show_labels)) * legend\n", + "labels = hv.DynamicMap(pn.bind(label_fn, enable=show_labels))\n", + "\n", + "overlay = tiles * pointsp * highlight * labels * legend\n", " \n", "pn.Column(\"# Categorical plot of AIS data by type\",\n", " \"Zoom or pan to explore the data, then click to select \"\n", From 42a4912281028539e30bbd3a2d9e803aa9d4fb9e Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 20 Jan 2021 16:59:28 -0600 Subject: [PATCH 16/46] Renamed ship_tracking to ship_traffic --- ship_traffic/AIS_categories.csv | 282 ++++++++++++++++++ ship_traffic/anaconda-project.yml | 64 ++++ .../ship_traffic.ipynb | 0 3 files changed, 346 insertions(+) create mode 100644 ship_traffic/AIS_categories.csv create mode 100644 ship_traffic/anaconda-project.yml rename ship_tracking/ship_tracking.ipynb => ship_traffic/ship_traffic.ipynb (100%) diff --git a/ship_traffic/AIS_categories.csv b/ship_traffic/AIS_categories.csv new file mode 100644 index 000000000..630270c4a --- /dev/null +++ b/ship_traffic/AIS_categories.csv @@ -0,0 +1,282 @@ +num,desc,category,category_desc +0,Not available,0,Unknown +1,Reserved,0,Unknown +2,Reserved,0,Unknown +3,Reserved,0,Unknown +4,Reserved,0,Unknown +5,Reserved,0,Unknown +6,Reserved,0,Unknown +7,Reserved,0,Unknown +8,Reserved,0,Unknown +9,Reserved,0,Unknown +10,Reserved,0,Unknown +11,Reserved,0,Unknown +12,Reserved,0,Unknown +13,Reserved,0,Unknown +14,Reserved,0,Unknown +15,Reserved,0,Unknown +16,Reserved,0,Unknown +17,Reserved,0,Unknown +18,Reserved,0,Unknown +19,Reserved,0,Unknown +20,"Wing in ground (WIG), all ships of this type",1,WIG +21,"Wing in ground (WIG), Hazardous category A",1,WIG +22,"Wing in ground (WIG), Hazardous category B",1,WIG +23,"Wing in ground (WIG), Hazardous category C",1,WIG +24,"Wing in ground (WIG), Hazardous category D",1,WIG +25,"Wing in ground (WIG), Reserved for future use",1,WIG +26,"Wing in ground (WIG), Reserved for future use",1,WIG +27,"Wing in ground (WIG), Reserved for future use",1,WIG +28,"Wing in ground (WIG), Reserved for future use",1,WIG +29,"Wing in ground (WIG), Reserved for future use",1,WIG +30,Fishing,2,Fishing +31,Towing,3,Towing +32,Towing: length exceeds 200m or breadth exceeds 25m,3,Towing +33,Dredging or underwater ops,4,Dredging +34,Diving ops,5,Diving +35,Military ops,6,Military +36,Sailing,7,Sailing +37,Pleasure Craft,8,Pleasure +38,Reserved,0,Unknown +39,Reserved,0,Unknown +40,"High speed craft (HSC), all ships of this type",9,High Speed +41,"High speed craft (HSC), Hazardous category A",9,High Speed +42,"High speed craft (HSC), Hazardous category B",9,High Speed +43,"High speed craft (HSC), Hazardous category C",9,High Speed +44,"High speed craft (HSC), Hazardous category D",9,High Speed +45,"High speed craft (HSC), Reserved for future use",9,High Speed +46,"High speed craft (HSC), Reserved for future use",9,High Speed +47,"High speed craft (HSC), Reserved for future use",9,High Speed +48,"High speed craft (HSC), Reserved for future use",9,High Speed +49,"High speed craft (HSC), No additional information",9,High Speed +50,Pilot Vessel,10,Pilot Vessel +51,Search and Rescue vessel,11,Search and Rescue vessel +52,Tug,12,Tug +53,Port Tender,18,Passenger +54,Anti-pollution equipment,13,Industrial +55,Law Enforcement,14,Law Enforcement +56,Spare - Local Vessel,15,Spare +57,Spare - Local Vessel,15,Spare +58,Medical Transport,16,Medical Transport +59,Noncombatant ship according to RR Resolution No. 18,17,Noncombatant +60,"Passenger, all ships of this type",18,Passenger +61,"Passenger, Hazardous category A",18,Passenger +62,"Passenger, Hazardous category B",18,Passenger +63,"Passenger, Hazardous category C",18,Passenger +64,"Passenger, Hazardous category D",18,Passenger +65,"Passenger, Reserved for future use",18,Passenger +66,"Passenger, Reserved for future use",18,Passenger +67,"Passenger, Reserved for future use",18,Passenger +68,"Passenger, Reserved for future use",18,Passenger +69,"Passenger, No additional information",18,Passenger +70,"Cargo, all ships of this type",19,Cargo +71,"Cargo, Hazardous category A",19,Cargo +72,"Cargo, Hazardous category B",19,Cargo +73,"Cargo, Hazardous category C",19,Cargo +74,"Cargo, Hazardous category D",19,Cargo +75,"Cargo, Reserved for future use",19,Cargo +76,"Cargo, Reserved for future use",19,Cargo +77,"Cargo, Reserved for future use",19,Cargo +78,"Cargo, Reserved for future use",19,Cargo +79,"Cargo, No additional information",19,Cargo +80,"Tanker, all ships of this type",20,Tanker +81,"Tanker, Hazardous category A",20,Tanker +82,"Tanker, Hazardous category B",20,Tanker +83,"Tanker, Hazardous category C",20,Tanker +84,"Tanker, Hazardous category D",20,Tanker +85,"Tanker, Reserved for future use",20,Tanker +86,"Tanker, Reserved for future use",20,Tanker +87,"Tanker, Reserved for future use",20,Tanker +88,"Tanker, Reserved for future use",20,Tanker +89,"Tanker, No additional information",20,Tanker +90,"Other Type, all ships of this type",21,Other +91,"Other Type, Hazardous category A",21,Other +92,"Other Type, Hazardous category B",21,Other +93,"Other Type, Hazardous category C",21,Other +94,"Other Type, Hazardous category D",21,Other +95,"Other Type, Reserved for future use",21,Other +96,"Other Type, Reserved for future use",21,Other +97,"Other Type, Reserved for future use",21,Other +98,"Other Type, Reserved for future use",21,Other +99,"Other Type, no additional information",21,Other +100,Reserved for future use,0,Unknown +101,Reserved for future use,0,Unknown +102,Reserved for future use,0,Unknown +103,Reserved for future use,0,Unknown +104,Reserved for future use,0,Unknown +105,Reserved for future use,0,Unknown +106,Reserved for future use,0,Unknown +107,Reserved for future use,0,Unknown +108,Reserved for future use,0,Unknown +109,Reserved for future use,0,Unknown +110,Reserved for future use,0,Unknown +111,Reserved for future use,0,Unknown +112,Reserved for future use,0,Unknown +113,Reserved for future use,0,Unknown +114,Reserved for future use,0,Unknown +115,Reserved for future use,0,Unknown +116,Reserved for future use,0,Unknown +117,Reserved for future use,0,Unknown +118,Reserved for future use,0,Unknown +119,Reserved for future use,0,Unknown +120,Reserved for future use,0,Unknown +121,Reserved for future use,0,Unknown +122,Reserved for future use,0,Unknown +123,Reserved for future use,0,Unknown +124,Reserved for future use,0,Unknown +125,Reserved for future use,0,Unknown +126,Reserved for future use,0,Unknown +127,Reserved for future use,0,Unknown +128,Reserved for future use,0,Unknown +129,Reserved for future use,0,Unknown +130,Reserved for future use,0,Unknown +131,Reserved for future use,0,Unknown +132,Reserved for future use,0,Unknown +133,Reserved for future use,0,Unknown +134,Reserved for future use,0,Unknown +135,Reserved for future use,0,Unknown +136,Reserved for future use,0,Unknown +137,Reserved for future use,0,Unknown +138,Reserved for future use,0,Unknown +139,Reserved for future use,0,Unknown +140,Reserved for future use,0,Unknown +141,Reserved for future use,0,Unknown +142,Reserved for future use,0,Unknown +143,Reserved for future use,0,Unknown +144,Reserved for future use,0,Unknown +145,Reserved for future use,0,Unknown +146,Reserved for future use,0,Unknown +147,Reserved for future use,0,Unknown +148,Reserved for future use,0,Unknown +149,Reserved for future use,0,Unknown +150,Reserved for future use,0,Unknown +151,Reserved for future use,0,Unknown +152,Reserved for future use,0,Unknown +153,Reserved for future use,0,Unknown +154,Reserved for future use,0,Unknown +155,Reserved for future use,0,Unknown +156,Reserved for future use,0,Unknown +157,Reserved for future use,0,Unknown +158,Reserved for future use,0,Unknown +159,Reserved for future use,0,Unknown +160,Reserved for future use,0,Unknown +161,Reserved for future use,0,Unknown +162,Reserved for future use,0,Unknown +163,Reserved for future use,0,Unknown +164,Reserved for future use,0,Unknown +165,Reserved for future use,0,Unknown +166,Reserved for future use,0,Unknown +167,Reserved for future use,0,Unknown +168,Reserved for future use,0,Unknown +169,Reserved for future use,0,Unknown +170,Reserved for future use,0,Unknown +171,Reserved for future use,0,Unknown +172,Reserved for future use,0,Unknown +173,Reserved for future use,0,Unknown +174,Reserved for future use,0,Unknown +175,Reserved for future use,0,Unknown +176,Reserved for future use,0,Unknown +177,Reserved for future use,0,Unknown +178,Reserved for future use,0,Unknown +179,Reserved for future use,0,Unknown +180,Reserved for future use,0,Unknown +181,Reserved for future use,0,Unknown +182,Reserved for future use,0,Unknown +183,Reserved for future use,0,Unknown +184,Reserved for future use,0,Unknown +185,Reserved for future use,0,Unknown +186,Reserved for future use,0,Unknown +187,Reserved for future use,0,Unknown +188,Reserved for future use,0,Unknown +189,Reserved for future use,0,Unknown +190,Reserved for future use,0,Unknown +191,Reserved for future use,0,Unknown +192,Reserved for future use,0,Unknown +193,Reserved for future use,0,Unknown +194,Reserved for future use,0,Unknown +195,Reserved for future use,0,Unknown +196,Reserved for future use,0,Unknown +197,Reserved for future use,0,Unknown +198,Reserved for future use,0,Unknown +199,Reserved for future use,0,Unknown +200,Reserved for future use,0,Unknown +201,Reserved for future use,0,Unknown +202,Reserved for future use,0,Unknown +203,Reserved for future use,0,Unknown +204,Reserved for future use,0,Unknown +205,Reserved for future use,0,Unknown +206,Reserved for future use,0,Unknown +207,Reserved for future use,0,Unknown +208,Reserved for future use,0,Unknown +209,Reserved for future use,0,Unknown +210,Reserved for future use,0,Unknown +211,Reserved for future use,0,Unknown +212,Reserved for future use,0,Unknown +213,Reserved for future use,0,Unknown +214,Reserved for future use,0,Unknown +215,Reserved for future use,0,Unknown +216,Reserved for future use,0,Unknown +217,Reserved for future use,0,Unknown +218,Reserved for future use,0,Unknown +219,Reserved for future use,0,Unknown +220,Reserved for future use,0,Unknown +221,Reserved for future use,0,Unknown +222,Reserved for future use,0,Unknown +223,Reserved for future use,0,Unknown +224,Reserved for future use,0,Unknown +225,Reserved for future use,0,Unknown +226,Reserved for future use,0,Unknown +227,Reserved for future use,0,Unknown +228,Reserved for future use,0,Unknown +229,Reserved for future use,0,Unknown +230,Reserved for future use,0,Unknown +231,Reserved for future use,0,Unknown +232,Reserved for future use,0,Unknown +233,Reserved for future use,0,Unknown +234,Reserved for future use,0,Unknown +235,Reserved for future use,0,Unknown +236,Reserved for future use,0,Unknown +237,Reserved for future use,0,Unknown +238,Reserved for future use,0,Unknown +239,Reserved for future use,0,Unknown +240,Reserved for future use,0,Unknown +241,Reserved for future use,0,Unknown +242,Reserved for future use,0,Unknown +243,Reserved for future use,0,Unknown +244,Reserved for future use,0,Unknown +245,Reserved for future use,0,Unknown +246,Reserved for future use,0,Unknown +247,Reserved for future use,0,Unknown +248,Reserved for future use,0,Unknown +249,Reserved for future use,0,Unknown +250,Reserved for future use,0,Unknown +251,Reserved for future use,0,Unknown +252,Reserved for future use,0,Unknown +253,Reserved for future use,0,Unknown +254,Reserved for future use,0,Unknown +255,Reserved for future use,0,Unknown +1001,Fishing vessels,2,Fishing +1002,Fishing vessels,2,Fishing +1003,Freight Vessels,19,Cargo +1004,Freight Vessels,19,Cargo +1005,Industrial vessels,13,Industrial +1006,Miscellaneous vessels,21,Other +1007,Offshore drilling vessels,13,Industrial +1008,non-vessel,21,Other +1009,non-vessel,21,Other +1010,Offshore supply vessel,13,Industrial +1011,Oil Recovery vessel,13,Industrial +1012,Passenger ships,18,Passenger +1013,Passenger ships,18,Passenger +1014,Passenger ships,18,Passenger +1015,Passenger ships,18,Passenger +1016,Public freight,19,Cargo +1017,Public tankship/barge,20,Tanker +1018,Unclassified public vessel,0,Unknown +1019,Recreational Vessel,8,Pleasure +1020,Research Vessel,21,Other +1021,SAR Aircraft,21,Other +1022,School ship,21,Other +1023,Tank Barge,20,Tanker +1024,Tank Ship,20,Tanker +1025,Towing Vessel,3,Towing diff --git a/ship_traffic/anaconda-project.yml b/ship_traffic/anaconda-project.yml new file mode 100644 index 000000000..972c7cdca --- /dev/null +++ b/ship_traffic/anaconda-project.yml @@ -0,0 +1,64 @@ +# To reproduce: install 'anaconda-project', then 'anaconda-project run' +name: ship_tracking +description: Visualizing AIS tracking data for ships near the USA +maintainers: +- jbednar +labels: +- datashader +- holoviews + +user_fields: [labels, skip, maintainers] + +channels: + - pyviz + +packages: &pkgs + - bokeh ==2.2.3 + - colorcet ==2 + - dask ==2020.12.0 + - datashader ==0.12.0 + - holoviews ==1.14.0 + - notebook ==6.1.5 + - numba ==0.51.2 + - numexpr ==2.7.1 + - pandas ==1.1.5 + - panel ==0.10.3 + - python ==3.7.9 + - spatialpandas ==0.3.6 + - xarray ==0.16.2 + - pip ==20.3.3 + - conda-forge::pyarrow ==2 + +dependencies: *pkgs + +commands: + dashboard: + unix: panel serve ship_tracking.ipynb + supports_http_options: true + notebook: + notebook: ship_tracking.ipynb + test: + unix: pytest --nbsmoke-run -k *.ipynb --ignore envs + windows: pytest --nbsmoke-run -k *.ipynb --ignore envs + env_spec: test + lint: + unix: pytest --nbsmoke-lint -k *.ipynb --ignore envs + windows: pytest --nbsmoke-lint -k *.ipynb --ignore envs + env_spec: test + +variables: {} +downloads: + DATAFILE: + url: https://zenodo.org/record/3541812/files/HG_OOSTENDE-gps-2018.csv + filename: data/HG_OOSTENDE-gps-2018.csv + +env_specs: + default: {} + test: + packages: + - nbsmoke=0.2.8 + - pytest=4.4.1 +platforms: +- linux-64 +- osx-64 +- win-64 diff --git a/ship_tracking/ship_tracking.ipynb b/ship_traffic/ship_traffic.ipynb similarity index 100% rename from ship_tracking/ship_tracking.ipynb rename to ship_traffic/ship_traffic.ipynb From c1e860268be8861180a0c88d93a61cc5d894d92b Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 20 Jan 2021 17:13:12 -0600 Subject: [PATCH 17/46] Made message match behavior --- ship_traffic/ship_traffic.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 95a09b960..b420585ea 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -283,8 +283,8 @@ " \n", "pn.Column(\"# Categorical plot of AIS data by type\",\n", " \"Zoom or pan to explore the data, then click to select \"\n", - " \"and highlight connected vessel tracks in a region. \",\n", - " \"You may need to zoom in before a track is selectable.\",\n", + " \"a particular data point to see more information about it (after a delay). \",\n", + " \"You may need to zoom in before a point is selectable.\",\n", " show_labels, overlay, sizing_mode='stretch_width').servable()" ] } From f48177924ffdabbb7cd1c13e632ec88a6b4ee347 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Thu, 21 Jan 2021 00:27:37 +0000 Subject: [PATCH 18/46] Added points_transformer to join data in Points hover --- ship_traffic/ship_traffic.ipynb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index b420585ea..25feb574a 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -245,6 +245,18 @@ "To help understand clusters of datapoints or individual datapoints, we can use the x,y location of a tap to query the dataset for a ping in that region, then highlight it on top of the main plot." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vessels_df = vessels.compute()\n", + "\n", + "def points_transformer(df):\n", + " return df.merge(vessels_df, on='MMSI').merge(vessel_types, on='category')" + ] + }, { "cell_type": "code", "execution_count": null, @@ -255,7 +267,8 @@ "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", "pointsp = dynspread(datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", - "highlight = inspect_points(pointsp, streams=[hv.streams.Tap], link_inputs=False)\n", + "vdims = ['MMSI', 'VesselName', 'Length', 'Width', 'category_desc']\n", + "highlight = inspect_points(pointsp, streams=[hv.streams.Tap], points_transformer=points_transformer, link_inputs=False, vdims=vdims)\n", "highlight = highlight.opts(color='white', tools=[\"hover\"], marker='square', size=10, fill_alpha=0)\n", "\n", "#tiles * pointsp * highlight * legend" From 7b4f163a40f5a50db659fa30df3dc40c222de14f Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 21 Jan 2021 19:27:45 +0100 Subject: [PATCH 19/46] Improve ship_traffic --- ship_traffic/ship_traffic.ipynb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 25feb574a..92d0a5b3e 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -167,25 +167,27 @@ " vessels = dd.read_parquet(vessels_file)\n", "\n", " print('Reading parquet file')\n", - " gdf = sp.io.read_parquet_dask(cache_file)\n", + " gdf = sp.io.read_parquet_dask(cache_file).persist()\n", " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", " \n", " else:\n", - " df = dd.read_csv(basedir+basename+'*.csv', usecols=dfcols+vesselcols, assume_missing=True)\n", + " csvs = basedir+basename+'*.csv'\n", + " df = dd.read_csv(csvs, usecols=vesselcols, assume_missing=True)\n", + " gdf = dd.read_csv(csvs, usecols=dfcols, assume_missing=True)\n", " with ProgressBar():\n", - " print('Reading CSV files')\n", - " gdf = df.map_partitions(convert_partition, meta=example).persist()\n", - "\n", " print('Writing vessel info file')\n", - " vessels = df[vesselcols].sample(frac=1).drop_duplicates([index]).compute()\n", + " vessels = df.groupby(index).last().reset_index().compute()\n", " vessels[index] = vessels[index].astype('int32')\n", " vessels.to_parquet(vessels_file)\n", "\n", + " print('Reading CSV files')\n", + " gdf = gdf.map_partitions(convert_partition, meta=example).persist()\n", + "\n", " print('Writing parquet file')\n", - " gdf = gdf.pack_partitions_to_parquet(cache_file, npartitions=64)\n", + " gdf = gdf.pack_partitions_to_parquet(cache_file, npartitions=64).persist()\n", " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", " \n", - " return gdf.persist(), vessels" + " return gdf, vessels" ] }, { @@ -268,7 +270,7 @@ "pointsp = dynspread(datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", "vdims = ['MMSI', 'VesselName', 'Length', 'Width', 'category_desc']\n", - "highlight = inspect_points(pointsp, streams=[hv.streams.Tap], points_transformer=points_transformer, link_inputs=False, vdims=vdims)\n", + "highlight = inspect_points(pointsp, streams=[hv.streams.Tap], points_transformer=points_transformer)\n", "highlight = highlight.opts(color='white', tools=[\"hover\"], marker='square', size=10, fill_alpha=0)\n", "\n", "#tiles * pointsp * highlight * legend" @@ -293,7 +295,7 @@ "labels = hv.DynamicMap(pn.bind(label_fn, enable=show_labels))\n", "\n", "overlay = tiles * pointsp * highlight * labels * legend\n", - " \n", + " \n", "pn.Column(\"# Categorical plot of AIS data by type\",\n", " \"Zoom or pan to explore the data, then click to select \"\n", " \"a particular data point to see more information about it (after a delay). \",\n", From 05d6e880f106bf0b67a88f8f89cfdb8ae2c12a8e Mon Sep 17 00:00:00 2001 From: jlstevens Date: Thu, 21 Jan 2021 21:53:33 +0000 Subject: [PATCH 20/46] Filtering vdims in transformer functions --- ship_traffic/ship_traffic.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 92d0a5b3e..234c0f570 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -256,7 +256,8 @@ "vessels_df = vessels.compute()\n", "\n", "def points_transformer(df):\n", - " return df.merge(vessels_df, on='MMSI').merge(vessel_types, on='category')" + " columns = ['geometry', 'MMSI', 'VesselName', 'Length', 'Width', 'category_desc']\n", + " return df.merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[columns]" ] }, { @@ -269,7 +270,6 @@ "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", "pointsp = dynspread(datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", - "vdims = ['MMSI', 'VesselName', 'Length', 'Width', 'category_desc']\n", "highlight = inspect_points(pointsp, streams=[hv.streams.Tap], points_transformer=points_transformer)\n", "highlight = highlight.opts(color='white', tools=[\"hover\"], marker='square', size=10, fill_alpha=0)\n", "\n", From 6e6a8daedce089c486e6b97a378986cf5f641deb Mon Sep 17 00:00:00 2001 From: jlstevens Date: Thu, 21 Jan 2021 23:00:04 +0000 Subject: [PATCH 21/46] Added drilldown table with URL --- ship_traffic/ship_traffic.ipynb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 234c0f570..0ff53f0b5 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -254,10 +254,16 @@ "outputs": [], "source": [ "vessels_df = vessels.compute()\n", + "columns = ['MMSI', 'VesselName', 'VesselType', 'Length', 'Width', 'category_desc']\n", "\n", "def points_transformer(df):\n", - " columns = ['geometry', 'MMSI', 'VesselName', 'Length', 'Width', 'category_desc']\n", - " return df.merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[columns]" + " return df.merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]\n", + "\n", + "def hits_transformer(df):\n", + " match = pd.DataFrame(points_transformer(df)[columns].iloc[0:1])\n", + " if len(match)==1: match['URL'] = pd.Series([\"https://www.marinetraffic.com/cs/ais/details/ships/mmsi:%d\" % match.iloc[0]['MMSI']])\n", + " else: match['URL'] = pd.Series([\"NaN\"])\n", + " return match" ] }, { @@ -270,7 +276,8 @@ "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", "pointsp = dynspread(datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", - "highlight = inspect_points(pointsp, streams=[hv.streams.Tap], points_transformer=points_transformer)\n", + "highlighter = inspect_points.instance(streams=[hv.streams.Tap], points_transformer=points_transformer, hits_transformer=hits_transformer)\n", + "highlight = highlighter(pointsp)\n", "highlight = highlight.opts(color='white', tools=[\"hover\"], marker='square', size=10, fill_alpha=0)\n", "\n", "#tiles * pointsp * highlight * legend" @@ -300,7 +307,9 @@ " \"Zoom or pan to explore the data, then click to select \"\n", " \"a particular data point to see more information about it (after a delay). \",\n", " \"You may need to zoom in before a point is selectable.\",\n", - " show_labels, overlay, sizing_mode='stretch_width').servable()" + " show_labels, overlay, \n", + " pn.Row(pn.Spacer(width=100), pn.bind(pn.pane.DataFrame, object=highlighter.param.hits, width=1200, index=False, render_links=True)),\n", + " sizing_mode='stretch_width').servable()" ] } ], From 3e2b1a95563eefdec5fb155af5bfd452885c8fe3 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Thu, 21 Jan 2021 22:00:40 -0600 Subject: [PATCH 22/46] Improved selection table and added sliders --- ship_traffic/ship_traffic.ipynb | 47 ++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 0ff53f0b5..45f0460b8 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -15,7 +15,7 @@ "metadata": {}, "outputs": [], "source": [ - "import os, numpy as np, pandas as pd, panel as pn, colorcet as cc, datashader as ds, holoviews as hv\n", + "import os, requests, numpy as np, pandas as pd, panel as pn, colorcet as cc, datashader as ds, holoviews as hv\n", "import spatialpandas as sp, spatialpandas.io, spatialpandas.geometry, spatialpandas.dask, dask.dataframe as dd\n", "\n", "from glob import glob\n", @@ -224,7 +224,7 @@ "metadata": {}, "outputs": [], "source": [ - "x_range, y_range = ll2en([-54,-128], [15,56])\n", + "x_range, y_range = ll2en([-54,-132], [15,51])\n", "bounds = dict(x=tuple(x_range), y=tuple(y_range))\n", "\n", "pts = hv.Points(df, vdims=['category']).redim.range(**bounds)\n", @@ -254,15 +254,21 @@ "outputs": [], "source": [ "vessels_df = vessels.compute()\n", - "columns = ['MMSI', 'VesselName', 'VesselType', 'Length', 'Width', 'category_desc']\n", + "columns = ['MMSI', 'IMO', 'VesselName', 'VesselType']\n", "\n", "def points_transformer(df):\n", " return df.merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]\n", "\n", + "ship_url = \"https://www.marinetraffic.com/cs/ais/details/ships/mmsi:\"\n", "def hits_transformer(df):\n", " match = pd.DataFrame(points_transformer(df)[columns].iloc[0:1])\n", - " if len(match)==1: match['URL'] = pd.Series([\"https://www.marinetraffic.com/cs/ais/details/ships/mmsi:%d\" % match.iloc[0]['MMSI']])\n", - " else: match['URL'] = pd.Series([\"NaN\"])\n", + " url = vdesc = \"\"\n", + " if len(match)==1:\n", + " row = match.iloc[0]\n", + " url = f\"{ship_url}{row.MMSI}\" \n", + " vdesc = f\"{int(row.VesselType)} ({vessel_types.loc[row.VesselType].desc})\"\n", + " match['URL'] = pd.Series([url])\n", + " match['VesselType'] = pd.Series([vdesc])\n", " return match" ] }, @@ -276,9 +282,11 @@ "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", "pointsp = dynspread(datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", - "highlighter = inspect_points.instance(streams=[hv.streams.Tap], points_transformer=points_transformer, hits_transformer=hits_transformer)\n", - "highlight = highlighter(pointsp)\n", - "highlight = highlight.opts(color='white', tools=[\"hover\"], marker='square', size=10, fill_alpha=0)\n", + "highlighter = inspect_points.instance(streams=[hv.streams.Tap], \n", + " points_transformer=points_transformer, \n", + " hits_transformer=hits_transformer)\n", + "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", + " size=10, fill_alpha=0)\n", "\n", "#tiles * pointsp * highlight * legend" ] @@ -287,7 +295,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We could view the result above by uncommenting the last line, but let's just go ahead and make a little app so that we can let the user decide whether to have labels visible:" + "We could view the result above by uncommenting the last line, but let's just go ahead and make a little app with widgets for controlling the visibility of the background map, the data, and the text labels, plus a table showing information about the selected vessel:" ] }, { @@ -296,20 +304,23 @@ "metadata": {}, "outputs": [], "source": [ - "def label_fn(enable=True):\n", - " return hv.element.tiles.StamenLabels().opts(level='glyph', alpha=0.9 if enable else 0)\n", - "show_labels = pn.widgets.Checkbox(name=\"Show labels\", value=True)\n", - "labels = hv.DynamicMap(pn.bind(label_fn, enable=show_labels))\n", + "map_opacity = pn.widgets.FloatSlider(start=0, end=1, value=0.7, name=\"Map opacity\")\n", + "data_opacity = pn.widgets.FloatSlider(start=0, end=1, value=1.0, name=\"Data opacity\")\n", + "label_opacity = pn.widgets.FloatSlider(start=0, end=1, value=0.9, name=\"Label opacity\")\n", "\n", - "overlay = tiles * pointsp * highlight * labels * legend\n", + "overlay = (tiles.apply.opts(alpha=map_opacity) * \n", + " pointsp.apply.opts(alpha=data_opacity) * \n", + " labels.apply.opts(alpha=label_opacity) * highlight * legend)\n", " \n", + "table = pn.bind(pn.pane.DataFrame, object=highlighter.param.hits, \n", + " width=1200, index=False, render_links=True, na_rep='')\n", + "\n", "pn.Column(\"# Categorical plot of AIS data by type\",\n", " \"Zoom or pan to explore the data, then click to select \"\n", - " \"a particular data point to see more information about it (after a delay). \",\n", + " \"a particular data point to see more information about it (after a delay). \"\n", " \"You may need to zoom in before a point is selectable.\",\n", - " show_labels, overlay, \n", - " pn.Row(pn.Spacer(width=100), pn.bind(pn.pane.DataFrame, object=highlighter.param.hits, width=1200, index=False, render_links=True)),\n", - " sizing_mode='stretch_width').servable()" + " pn.Row(map_opacity, data_opacity, label_opacity),\n", + " overlay, table, sizing_mode='stretch_width').servable()" ] } ], From 0882d893f02ed714e25c82d0e35abf54d30ce241 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Fri, 22 Jan 2021 14:40:13 -0600 Subject: [PATCH 23/46] Removed duplicate versions of files --- ship_tracking/AIS_categories.csv | 282 ----------------------------- ship_tracking/anaconda-project.yml | 64 ------- 2 files changed, 346 deletions(-) delete mode 100644 ship_tracking/AIS_categories.csv delete mode 100644 ship_tracking/anaconda-project.yml diff --git a/ship_tracking/AIS_categories.csv b/ship_tracking/AIS_categories.csv deleted file mode 100644 index 630270c4a..000000000 --- a/ship_tracking/AIS_categories.csv +++ /dev/null @@ -1,282 +0,0 @@ -num,desc,category,category_desc -0,Not available,0,Unknown -1,Reserved,0,Unknown -2,Reserved,0,Unknown -3,Reserved,0,Unknown -4,Reserved,0,Unknown -5,Reserved,0,Unknown -6,Reserved,0,Unknown -7,Reserved,0,Unknown -8,Reserved,0,Unknown -9,Reserved,0,Unknown -10,Reserved,0,Unknown -11,Reserved,0,Unknown -12,Reserved,0,Unknown -13,Reserved,0,Unknown -14,Reserved,0,Unknown -15,Reserved,0,Unknown -16,Reserved,0,Unknown -17,Reserved,0,Unknown -18,Reserved,0,Unknown -19,Reserved,0,Unknown -20,"Wing in ground (WIG), all ships of this type",1,WIG -21,"Wing in ground (WIG), Hazardous category A",1,WIG -22,"Wing in ground (WIG), Hazardous category B",1,WIG -23,"Wing in ground (WIG), Hazardous category C",1,WIG -24,"Wing in ground (WIG), Hazardous category D",1,WIG -25,"Wing in ground (WIG), Reserved for future use",1,WIG -26,"Wing in ground (WIG), Reserved for future use",1,WIG -27,"Wing in ground (WIG), Reserved for future use",1,WIG -28,"Wing in ground (WIG), Reserved for future use",1,WIG -29,"Wing in ground (WIG), Reserved for future use",1,WIG -30,Fishing,2,Fishing -31,Towing,3,Towing -32,Towing: length exceeds 200m or breadth exceeds 25m,3,Towing -33,Dredging or underwater ops,4,Dredging -34,Diving ops,5,Diving -35,Military ops,6,Military -36,Sailing,7,Sailing -37,Pleasure Craft,8,Pleasure -38,Reserved,0,Unknown -39,Reserved,0,Unknown -40,"High speed craft (HSC), all ships of this type",9,High Speed -41,"High speed craft (HSC), Hazardous category A",9,High Speed -42,"High speed craft (HSC), Hazardous category B",9,High Speed -43,"High speed craft (HSC), Hazardous category C",9,High Speed -44,"High speed craft (HSC), Hazardous category D",9,High Speed -45,"High speed craft (HSC), Reserved for future use",9,High Speed -46,"High speed craft (HSC), Reserved for future use",9,High Speed -47,"High speed craft (HSC), Reserved for future use",9,High Speed -48,"High speed craft (HSC), Reserved for future use",9,High Speed -49,"High speed craft (HSC), No additional information",9,High Speed -50,Pilot Vessel,10,Pilot Vessel -51,Search and Rescue vessel,11,Search and Rescue vessel -52,Tug,12,Tug -53,Port Tender,18,Passenger -54,Anti-pollution equipment,13,Industrial -55,Law Enforcement,14,Law Enforcement -56,Spare - Local Vessel,15,Spare -57,Spare - Local Vessel,15,Spare -58,Medical Transport,16,Medical Transport -59,Noncombatant ship according to RR Resolution No. 18,17,Noncombatant -60,"Passenger, all ships of this type",18,Passenger -61,"Passenger, Hazardous category A",18,Passenger -62,"Passenger, Hazardous category B",18,Passenger -63,"Passenger, Hazardous category C",18,Passenger -64,"Passenger, Hazardous category D",18,Passenger -65,"Passenger, Reserved for future use",18,Passenger -66,"Passenger, Reserved for future use",18,Passenger -67,"Passenger, Reserved for future use",18,Passenger -68,"Passenger, Reserved for future use",18,Passenger -69,"Passenger, No additional information",18,Passenger -70,"Cargo, all ships of this type",19,Cargo -71,"Cargo, Hazardous category A",19,Cargo -72,"Cargo, Hazardous category B",19,Cargo -73,"Cargo, Hazardous category C",19,Cargo -74,"Cargo, Hazardous category D",19,Cargo -75,"Cargo, Reserved for future use",19,Cargo -76,"Cargo, Reserved for future use",19,Cargo -77,"Cargo, Reserved for future use",19,Cargo -78,"Cargo, Reserved for future use",19,Cargo -79,"Cargo, No additional information",19,Cargo -80,"Tanker, all ships of this type",20,Tanker -81,"Tanker, Hazardous category A",20,Tanker -82,"Tanker, Hazardous category B",20,Tanker -83,"Tanker, Hazardous category C",20,Tanker -84,"Tanker, Hazardous category D",20,Tanker -85,"Tanker, Reserved for future use",20,Tanker -86,"Tanker, Reserved for future use",20,Tanker -87,"Tanker, Reserved for future use",20,Tanker -88,"Tanker, Reserved for future use",20,Tanker -89,"Tanker, No additional information",20,Tanker -90,"Other Type, all ships of this type",21,Other -91,"Other Type, Hazardous category A",21,Other -92,"Other Type, Hazardous category B",21,Other -93,"Other Type, Hazardous category C",21,Other -94,"Other Type, Hazardous category D",21,Other -95,"Other Type, Reserved for future use",21,Other -96,"Other Type, Reserved for future use",21,Other -97,"Other Type, Reserved for future use",21,Other -98,"Other Type, Reserved for future use",21,Other -99,"Other Type, no additional information",21,Other -100,Reserved for future use,0,Unknown -101,Reserved for future use,0,Unknown -102,Reserved for future use,0,Unknown -103,Reserved for future use,0,Unknown -104,Reserved for future use,0,Unknown -105,Reserved for future use,0,Unknown -106,Reserved for future use,0,Unknown -107,Reserved for future use,0,Unknown -108,Reserved for future use,0,Unknown -109,Reserved for future use,0,Unknown -110,Reserved for future use,0,Unknown -111,Reserved for future use,0,Unknown -112,Reserved for future use,0,Unknown -113,Reserved for future use,0,Unknown -114,Reserved for future use,0,Unknown -115,Reserved for future use,0,Unknown -116,Reserved for future use,0,Unknown -117,Reserved for future use,0,Unknown -118,Reserved for future use,0,Unknown -119,Reserved for future use,0,Unknown -120,Reserved for future use,0,Unknown -121,Reserved for future use,0,Unknown -122,Reserved for future use,0,Unknown -123,Reserved for future use,0,Unknown -124,Reserved for future use,0,Unknown -125,Reserved for future use,0,Unknown -126,Reserved for future use,0,Unknown -127,Reserved for future use,0,Unknown -128,Reserved for future use,0,Unknown -129,Reserved for future use,0,Unknown -130,Reserved for future use,0,Unknown -131,Reserved for future use,0,Unknown -132,Reserved for future use,0,Unknown -133,Reserved for future use,0,Unknown -134,Reserved for future use,0,Unknown -135,Reserved for future use,0,Unknown -136,Reserved for future use,0,Unknown -137,Reserved for future use,0,Unknown -138,Reserved for future use,0,Unknown -139,Reserved for future use,0,Unknown -140,Reserved for future use,0,Unknown -141,Reserved for future use,0,Unknown -142,Reserved for future use,0,Unknown -143,Reserved for future use,0,Unknown -144,Reserved for future use,0,Unknown -145,Reserved for future use,0,Unknown -146,Reserved for future use,0,Unknown -147,Reserved for future use,0,Unknown -148,Reserved for future use,0,Unknown -149,Reserved for future use,0,Unknown -150,Reserved for future use,0,Unknown -151,Reserved for future use,0,Unknown -152,Reserved for future use,0,Unknown -153,Reserved for future use,0,Unknown -154,Reserved for future use,0,Unknown -155,Reserved for future use,0,Unknown -156,Reserved for future use,0,Unknown -157,Reserved for future use,0,Unknown -158,Reserved for future use,0,Unknown -159,Reserved for future use,0,Unknown -160,Reserved for future use,0,Unknown -161,Reserved for future use,0,Unknown -162,Reserved for future use,0,Unknown -163,Reserved for future use,0,Unknown -164,Reserved for future use,0,Unknown -165,Reserved for future use,0,Unknown -166,Reserved for future use,0,Unknown -167,Reserved for future use,0,Unknown -168,Reserved for future use,0,Unknown -169,Reserved for future use,0,Unknown -170,Reserved for future use,0,Unknown -171,Reserved for future use,0,Unknown -172,Reserved for future use,0,Unknown -173,Reserved for future use,0,Unknown -174,Reserved for future use,0,Unknown -175,Reserved for future use,0,Unknown -176,Reserved for future use,0,Unknown -177,Reserved for future use,0,Unknown -178,Reserved for future use,0,Unknown -179,Reserved for future use,0,Unknown -180,Reserved for future use,0,Unknown -181,Reserved for future use,0,Unknown -182,Reserved for future use,0,Unknown -183,Reserved for future use,0,Unknown -184,Reserved for future use,0,Unknown -185,Reserved for future use,0,Unknown -186,Reserved for future use,0,Unknown -187,Reserved for future use,0,Unknown -188,Reserved for future use,0,Unknown -189,Reserved for future use,0,Unknown -190,Reserved for future use,0,Unknown -191,Reserved for future use,0,Unknown -192,Reserved for future use,0,Unknown -193,Reserved for future use,0,Unknown -194,Reserved for future use,0,Unknown -195,Reserved for future use,0,Unknown -196,Reserved for future use,0,Unknown -197,Reserved for future use,0,Unknown -198,Reserved for future use,0,Unknown -199,Reserved for future use,0,Unknown -200,Reserved for future use,0,Unknown -201,Reserved for future use,0,Unknown -202,Reserved for future use,0,Unknown -203,Reserved for future use,0,Unknown -204,Reserved for future use,0,Unknown -205,Reserved for future use,0,Unknown -206,Reserved for future use,0,Unknown -207,Reserved for future use,0,Unknown -208,Reserved for future use,0,Unknown -209,Reserved for future use,0,Unknown -210,Reserved for future use,0,Unknown -211,Reserved for future use,0,Unknown -212,Reserved for future use,0,Unknown -213,Reserved for future use,0,Unknown -214,Reserved for future use,0,Unknown -215,Reserved for future use,0,Unknown -216,Reserved for future use,0,Unknown -217,Reserved for future use,0,Unknown -218,Reserved for future use,0,Unknown -219,Reserved for future use,0,Unknown -220,Reserved for future use,0,Unknown -221,Reserved for future use,0,Unknown -222,Reserved for future use,0,Unknown -223,Reserved for future use,0,Unknown -224,Reserved for future use,0,Unknown -225,Reserved for future use,0,Unknown -226,Reserved for future use,0,Unknown -227,Reserved for future use,0,Unknown -228,Reserved for future use,0,Unknown -229,Reserved for future use,0,Unknown -230,Reserved for future use,0,Unknown -231,Reserved for future use,0,Unknown -232,Reserved for future use,0,Unknown -233,Reserved for future use,0,Unknown -234,Reserved for future use,0,Unknown -235,Reserved for future use,0,Unknown -236,Reserved for future use,0,Unknown -237,Reserved for future use,0,Unknown -238,Reserved for future use,0,Unknown -239,Reserved for future use,0,Unknown -240,Reserved for future use,0,Unknown -241,Reserved for future use,0,Unknown -242,Reserved for future use,0,Unknown -243,Reserved for future use,0,Unknown -244,Reserved for future use,0,Unknown -245,Reserved for future use,0,Unknown -246,Reserved for future use,0,Unknown -247,Reserved for future use,0,Unknown -248,Reserved for future use,0,Unknown -249,Reserved for future use,0,Unknown -250,Reserved for future use,0,Unknown -251,Reserved for future use,0,Unknown -252,Reserved for future use,0,Unknown -253,Reserved for future use,0,Unknown -254,Reserved for future use,0,Unknown -255,Reserved for future use,0,Unknown -1001,Fishing vessels,2,Fishing -1002,Fishing vessels,2,Fishing -1003,Freight Vessels,19,Cargo -1004,Freight Vessels,19,Cargo -1005,Industrial vessels,13,Industrial -1006,Miscellaneous vessels,21,Other -1007,Offshore drilling vessels,13,Industrial -1008,non-vessel,21,Other -1009,non-vessel,21,Other -1010,Offshore supply vessel,13,Industrial -1011,Oil Recovery vessel,13,Industrial -1012,Passenger ships,18,Passenger -1013,Passenger ships,18,Passenger -1014,Passenger ships,18,Passenger -1015,Passenger ships,18,Passenger -1016,Public freight,19,Cargo -1017,Public tankship/barge,20,Tanker -1018,Unclassified public vessel,0,Unknown -1019,Recreational Vessel,8,Pleasure -1020,Research Vessel,21,Other -1021,SAR Aircraft,21,Other -1022,School ship,21,Other -1023,Tank Barge,20,Tanker -1024,Tank Ship,20,Tanker -1025,Towing Vessel,3,Towing diff --git a/ship_tracking/anaconda-project.yml b/ship_tracking/anaconda-project.yml deleted file mode 100644 index 972c7cdca..000000000 --- a/ship_tracking/anaconda-project.yml +++ /dev/null @@ -1,64 +0,0 @@ -# To reproduce: install 'anaconda-project', then 'anaconda-project run' -name: ship_tracking -description: Visualizing AIS tracking data for ships near the USA -maintainers: -- jbednar -labels: -- datashader -- holoviews - -user_fields: [labels, skip, maintainers] - -channels: - - pyviz - -packages: &pkgs - - bokeh ==2.2.3 - - colorcet ==2 - - dask ==2020.12.0 - - datashader ==0.12.0 - - holoviews ==1.14.0 - - notebook ==6.1.5 - - numba ==0.51.2 - - numexpr ==2.7.1 - - pandas ==1.1.5 - - panel ==0.10.3 - - python ==3.7.9 - - spatialpandas ==0.3.6 - - xarray ==0.16.2 - - pip ==20.3.3 - - conda-forge::pyarrow ==2 - -dependencies: *pkgs - -commands: - dashboard: - unix: panel serve ship_tracking.ipynb - supports_http_options: true - notebook: - notebook: ship_tracking.ipynb - test: - unix: pytest --nbsmoke-run -k *.ipynb --ignore envs - windows: pytest --nbsmoke-run -k *.ipynb --ignore envs - env_spec: test - lint: - unix: pytest --nbsmoke-lint -k *.ipynb --ignore envs - windows: pytest --nbsmoke-lint -k *.ipynb --ignore envs - env_spec: test - -variables: {} -downloads: - DATAFILE: - url: https://zenodo.org/record/3541812/files/HG_OOSTENDE-gps-2018.csv - filename: data/HG_OOSTENDE-gps-2018.csv - -env_specs: - default: {} - test: - packages: - - nbsmoke=0.2.8 - - pytest=4.4.1 -platforms: -- linux-64 -- osx-64 -- win-64 From 7ba5494da639209353e159bd1076e161ba8dd86d Mon Sep 17 00:00:00 2001 From: jlstevens Date: Fri, 22 Jan 2021 23:34:23 +0000 Subject: [PATCH 24/46] Added photo pane --- ship_traffic/ship_traffic.ipynb | 35 +++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 45f0460b8..2033df808 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -17,6 +17,7 @@ "source": [ "import os, requests, numpy as np, pandas as pd, panel as pn, colorcet as cc, datashader as ds, holoviews as hv\n", "import spatialpandas as sp, spatialpandas.io, spatialpandas.geometry, spatialpandas.dask, dask.dataframe as dd\n", + "from PIL import Image\n", "\n", "from glob import glob\n", "from holoviews.util.transform import lon_lat_to_easting_northing as ll2en\n", @@ -304,14 +305,40 @@ "metadata": {}, "outputs": [], "source": [ + "def get_photo_url(mmsi):\n", + " headers = {'User-Agent': 'Mozilla/5.0'}\n", + " r=requests.get(f'https://www.marinetraffic.com/cs/ais/details/ships/mmsi:{mmsi}', \n", + " allow_redirects=True, headers=headers)\n", + " ship_id = [el for el in r.url.split('/') if el.startswith('shipid')]\n", + " if ship_id == []: return 'Not found'\n", + " ship_id =ship_id[0].replace('shipid:','')\n", + " return f\"https://photos.marinetraffic.com/ais/showphoto.aspx?shipid={ship_id}&size=thumb300&stamp=false\"\n", + "\n", + "def get_photo(hits=None):\n", + " try:\n", + " url = get_photo_url(hits.iloc[0]['MMSI'])\n", + " response = requests.get(url, stream=True)\n", + " im = Image.open(response.raw)\n", + " except: \n", + " im = Image.new('RGB', (1,1), (255, 255, 255))\n", + " return pn.panel(im)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "photo = pn.bind(get_photo, hits=highlighter.param.hits)\n", + "\n", "map_opacity = pn.widgets.FloatSlider(start=0, end=1, value=0.7, name=\"Map opacity\")\n", "data_opacity = pn.widgets.FloatSlider(start=0, end=1, value=1.0, name=\"Data opacity\")\n", "label_opacity = pn.widgets.FloatSlider(start=0, end=1, value=0.9, name=\"Label opacity\")\n", "\n", - "overlay = (tiles.apply.opts(alpha=map_opacity) * \n", - " pointsp.apply.opts(alpha=data_opacity) * \n", + "overlay = (tiles.apply.opts(alpha=map_opacity) *\n", + " pointsp.apply.opts(alpha=data_opacity) *\n", " labels.apply.opts(alpha=label_opacity) * highlight * legend)\n", - " \n", "table = pn.bind(pn.pane.DataFrame, object=highlighter.param.hits, \n", " width=1200, index=False, render_links=True, na_rep='')\n", "\n", @@ -320,7 +347,7 @@ " \"a particular data point to see more information about it (after a delay). \"\n", " \"You may need to zoom in before a point is selectable.\",\n", " pn.Row(map_opacity, data_opacity, label_opacity),\n", - " overlay, table, sizing_mode='stretch_width').servable()" + " overlay, table, photo, sizing_mode='stretch_width').servable()" ] } ], From 9f3f830eec74c1aef91cfe58b2fc88bd85e7b0da Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Fri, 22 Jan 2021 19:41:58 -0600 Subject: [PATCH 25/46] Updated text and comments. Made spatial indexing optional. Used tinyurl for formatting. --- ship_traffic/ship_traffic.ipynb | 154 ++++++++++++++++++++------------ 1 file changed, 99 insertions(+), 55 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 2033df808..75e013ad2 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -6,7 +6,7 @@ "source": [ "# Exploring AIS vessel-traffic data\n", "\n", - "This [Jupyter](https://jupyter.org) notebook demonstrates how to use the [Datashader](https://datashader.org)-based rendering in [HoloViews](https://holoviews.org) to explore and analyze US Coast Guard [Automatic Identification System (AIS)](https://en.wikipedia.org/wiki/Automatic_identification_system) vessel-location data. Vessels are identified by their [Maritime Mobile Service Identity](https://en.wikipedia.org/wiki/Maritime_Mobile_Service_Identity) numbers, and other data about the vessels is also typically included. Data is provided for January 2020, but additional months and years of data can be downloaded for US coastal areas from [marinecadastre.gov](marinehttps://marinecadastre.gov/ais), and similar approaches should be usable for other AIS data available for other regions." + "This [Jupyter](https://jupyter.org) notebook demonstrates how to use the [Datashader](https://datashader.org)-based rendering in [HoloViews](https://holoviews.org) to explore and analyze US Coast Guard [Automatic Identification System (AIS)](https://en.wikipedia.org/wiki/Automatic_identification_system) vessel-location data. AIS data includes vessels identified by their [Maritime Mobile Service Identity](https://en.wikipedia.org/wiki/Maritime_Mobile_Service_Identity) numbers along with other data such as vessel type. Data is provided here for January 2020 (200 million datapoints), but additional months and years of data can be downloaded for US coastal areas from [marinecadastre.gov](marinehttps://marinecadastre.gov/ais), and with slight modifications the same code here should work for AIS data available for other regions. This notebook also illustrates a workflow for visualizing large categorical datasets in general, letting users interact with individual datapoints even though the data itself is never sent to the browser for plotting." ] }, { @@ -15,13 +15,13 @@ "metadata": {}, "outputs": [], "source": [ - "import os, requests, numpy as np, pandas as pd, panel as pn, colorcet as cc, datashader as ds, holoviews as hv\n", - "import spatialpandas as sp, spatialpandas.io, spatialpandas.geometry, spatialpandas.dask, dask.dataframe as dd\n", - "from PIL import Image\n", + "import os, requests, numpy as np, pandas as pd, holoviews as hv, holoviews.operation.datashader as hd\n", + "import dask.dataframe as dd, panel as pn, panel.widgets as pnw, colorcet as cc, datashader as ds\n", + "import spatialpandas as sp, spatialpandas.io, spatialpandas.geometry, spatialpandas.dask\n", "\n", + "from PIL import Image\n", "from glob import glob\n", "from holoviews.util.transform import lon_lat_to_easting_northing as ll2en\n", - "from holoviews.operation.datashader import rasterize, datashade, dynspread, inspect_points\n", "from dask.diagnostics import ProgressBar\n", "\n", "hv.extension('bokeh', width=100)" @@ -33,9 +33,9 @@ "source": [ "## Vessel categories \n", "\n", - "AIS pings come with an associated integer `VesselType`, which broadly labels what sort of vessel it is. Different types of vessels are used for different purposes and behave differently, as we can see if we color-code the location of each ping by the `VesselType` using Datshader. \n", + "AIS pings come with an associated integer `VesselType`, which broadly labels what sort of vessel it is. Different types of vessels are used for different purposes and behave differently, as we will see below when we color-code the location of each ping by the `VesselType` using Datshader. \n", "\n", - "Type names are defined in a separate file constructed using lists of 100+ [AIS Vessel Types](https://api.vtexplorer.com/docs/ref-aistypes.html), and can be further collapsed into a smaller number of broad vessel categories:" + "Here, we'll use type names defined in a separate file constructed using lists of 100+ [AIS Vessel Types](https://api.vtexplorer.com/docs/ref-aistypes.html). We've further collapsed those types into a smaller number of vessel categories:" ] }, { @@ -52,7 +52,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can further reduce the `category` to the 6 most common (with the rest as `Other`). We will create a dictionary which maps the value to one of the categories:" + "We'll further reduce the `category` to the 6 most common, with the rest as `Other`." ] }, { @@ -75,7 +75,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now let us look at the categories:" + "We can print the resulting categories by number, with their description:" ] }, { @@ -92,7 +92,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Given a set of colors, let's construct a color key for Datashader to use later, along with a visible legend we can add to such a plot:" + "We'll map these categories to colors defined by [colorcet](https://colorcet.holoviz.org) and construct a color key and legend to use for plotting:" ] }, { @@ -105,9 +105,9 @@ "color_key = {list(groups.keys())[i]:tuple(int(e*255.) for e in v) for i,v in \n", " enumerate(colors[:(len(groups))][::-1])}\n", "legend = hv.NdOverlay({groups[k]: hv.Points([0,0], label=str(groups[k])).opts(\n", - " color=cc.rgb_to_hex(*v), size=0) \n", + " color=cc.rgb_to_hex(*v), size=0) \n", " for k, v in color_key.items()})\n", - "#legend #uncomment to see legend alone" + "#legend # uncomment to see legend alone" ] }, { @@ -116,9 +116,7 @@ "source": [ "## Load AIS pings\n", "\n", - "Next we will load the data from disk, either directly from a spatially indexed Parquet file (if previously cached) or from the raw CSV files. We'll also project the data to the coordinate system we will use later for plotting.\n", - "\n", - "Since particularly in raw form this is a lot of data, we will use the `map_partitions` functionality of a dask.DataFrame. To do this we define a function to the conversion and an example DataFrame with the required structure:" + "Next we will load the data from disk, either directly from a spatially indexed Parquet file (if previously cached) or from the raw CSV files. We'll also project the data to the coordinate system we will use later for plotting. There's a lot of data to process, so we'll use [Dask](https://dask.org) to ensure that we use all the cores available on this machine. Dask breaks a dataset into partitions that can be processed in parallel, so here we define a function for dealing with one partition's worth of data, along with a schema showing what the final dataframe's columnar structure will be." ] }, { @@ -144,7 +142,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Next we will define the function that will load our data, reading a much-smaller (and much faster to load) cached Parquet-format file from disk if available:" + "Next we will define the function that will load our data, reading a much-smaller (and much faster to load) previously cached Parquet-format file from disk if available. To use files covering other date ranges, just download them and change `basedir` and/or `basename` to match them." ] }, { @@ -159,7 +157,7 @@ "dfcols = ['MMSI', 'LON', 'LAT', 'BaseDateTime', 'VesselType']\n", "vesselcols = ['MMSI', 'IMO', 'CallSign', 'VesselName', 'VesselType', 'Length', 'Width']\n", "\n", - "def load_data():\n", + "def load_data(spatial_index=False):\n", " cache_file = basedir+basename+'_broadcast.parq'\n", " vessels_file = basedir+basename+'_vessels.parq'\n", " \n", @@ -169,25 +167,29 @@ "\n", " print('Reading parquet file')\n", " gdf = sp.io.read_parquet_dask(cache_file).persist()\n", - " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", " \n", " else:\n", " csvs = basedir+basename+'*.csv'\n", - " df = dd.read_csv(csvs, usecols=vesselcols, assume_missing=True)\n", - " gdf = dd.read_csv(csvs, usecols=dfcols, assume_missing=True)\n", " with ProgressBar():\n", " print('Writing vessel info file')\n", + " df = dd.read_csv(csvs, usecols=vesselcols, assume_missing=True)\n", " vessels = df.groupby(index).last().reset_index().compute()\n", " vessels[index] = vessels[index].astype('int32')\n", " vessels.to_parquet(vessels_file)\n", "\n", " print('Reading CSV files')\n", + " gdf = dd.read_csv(csvs, usecols=dfcols, assume_missing=True)\n", " gdf = gdf.map_partitions(convert_partition, meta=example).persist()\n", "\n", " print('Writing parquet file')\n", " gdf = gdf.pack_partitions_to_parquet(cache_file, npartitions=64).persist()\n", - " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", - " \n", + " \n", + " with ProgressBar(): \n", + " print('Building spatial index') # Takes a couple of minutes for 1 month's data\n", + " if spatial_index: gdf = gdf.build_sindex()\n", + " gdf = gdf.persist()\n", + " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", + "\n", " return gdf, vessels" ] }, @@ -195,7 +197,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Actually load the data, using the disk cache and memory cache if available:" + "Now let's actually load the data, using the disk cache and memory cache if available. You can change `spatial_index` to `True` above to speed up selection of individual points in the final plot, though `load_data` will then take several minutes rather than a few seconds." ] }, { @@ -208,6 +210,22 @@ "df, vessels = pn.state.as_cached('df', load_data)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we can see that this is a collection of points (latitude and longitude projected to Web Mercator) with an associated integer MMSI and category value:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.head()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -216,7 +234,7 @@ "\n", "We can now plot the data colored by category, with a color key.\n", "\n", - "To zoom in & interact with the plot, click the “Wheel zoom” tool in the toolbar on the side of the plot. Click and drag the plot in order to look around. As you zoom in, finer-grained detail will emerge and fill in, as long as you have a live Python process running to render the data dynamically. Depending on the size of the dataset and your machine, updating the plot might take a few seconds." + "To zoom in & interact with the plot, click the “Wheel zoom” tool in the [Bokeh toolbar](https://docs.bokeh.org/en/latest/docs/user_guide/tools.html) on the side of the plot and click and drag the plot in order to look around, or use the \"Box Zoom\" tool to select an area of interest. As you zoom in, finer-grained detail will emerge and fill in, as long as you have a live Python process running to render the data dynamically. Depending on the size of the dataset and your machine, updating the plot might take a few seconds." ] }, { @@ -229,7 +247,7 @@ "bounds = dict(x=tuple(x_range), y=tuple(y_range))\n", "\n", "pts = hv.Points(df, vdims=['category']).redim.range(**bounds)\n", - "points = dynspread(datashade(pts, aggregator=ds.count_cat('category'), color_key=color_key))\n", + "points = hd.dynspread(hd.datashade(pts, aggregator=ds.count_cat('category'), color_key=color_key))\n", "\n", "tiles = hv.element.tiles.ESRI().opts(alpha=0.4, bgcolor=\"black\").opts(responsive=True, min_height=600)\n", "labels = hv.element.tiles.StamenLabels().opts(alpha=0.7, level='glyph')\n", @@ -241,11 +259,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Clearly, the ship's behavior is highly dependent on category, with very different patterns of motion between these categories (and presumably the other categories not shown). E.g. passenger vessels tend to travel _across_ narrow waterways, while towing and cargo vessels travel _along_ them. Fishing vessels, as one would expect, travel out to open water and then cover a wide area around their initial destination. Zooming and panning (using the [Bokeh](https://docs.bokeh.org/en/latest/docs/user_guide/tools.html) tools at the right) reveal other patterns at different locations and scales.\n", + "Clearly, vessel behavior is highly dependent on category, with very different patterns of motion between these categories (and presumably the other categories not shown). E.g. fishing vessels tend to hug the coasts in meandering patterns, while cargo vessels travel along straight lines further from the coast. If you zoom in to a river, you can see that passenger vessels tend to travel _across_ narrow waterways, while towing and cargo vessels travel _along_ them. Fishing vessels, as one would expect, travel out to open water and then cover a wide area around their initial destination. Zooming and panning (using the tools at the right) reveal other patterns at different locations and scales.\n", "\n", "# Selecting specific datapoints\n", "\n", - "To help understand clusters of datapoints or individual datapoints, we can use the x,y location of a tap to query the dataset for a ping in that region, then highlight it on top of the main plot." + "Datashader renders data into a screen-sized array of values or pixels, which allows it to handle much larger volumes of data than can be sent to a web browser. What if you what to interact with the underlying data, e.g. to get information about clusters of datapoints or even individual datapoints? We can use HoloViews and Bokeh tools to watch for the x,y location of a tap, then query the underlying dataset for a ping in that region, and then then highlight it on top of the main plot." ] }, { @@ -255,19 +273,19 @@ "outputs": [], "source": [ "vessels_df = vessels.compute()\n", - "columns = ['MMSI', 'IMO', 'VesselName', 'VesselType']\n", + "columns = ['VesselName', 'MMSI', 'IMO', 'VesselType']\n", "\n", "def points_transformer(df):\n", " return df.merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]\n", "\n", - "ship_url = \"https://www.marinetraffic.com/cs/ais/details/ships/mmsi:\"\n", + "ship_url = 'https://tinyurl.com/aispage/mmsi:'\n", "def hits_transformer(df):\n", " match = pd.DataFrame(points_transformer(df)[columns].iloc[0:1])\n", - " url = vdesc = \"\"\n", + " url = vdesc = ''\n", " if len(match)==1:\n", " row = match.iloc[0]\n", - " url = f\"{ship_url}{row.MMSI}\" \n", - " vdesc = f\"{int(row.VesselType)} ({vessel_types.loc[row.VesselType].desc})\"\n", + " url = f'{ship_url}{row.MMSI}'\n", + " vdesc = f'{row.VesselType:.0f} ({vessel_types.loc[row.VesselType].desc})'\n", " match['URL'] = pd.Series([url])\n", " match['VesselType'] = pd.Series([vdesc])\n", " return match" @@ -281,11 +299,10 @@ "source": [ "xr, yr = ll2en([-126,-120.7], [47.5,49.5])\n", "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", - "pointsp = dynspread(datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", + "pointsp = hd.dynspread(hd.datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", - "highlighter = inspect_points.instance(streams=[hv.streams.Tap], \n", - " points_transformer=points_transformer, \n", - " hits_transformer=hits_transformer)\n", + "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], \n", + " points_transformer=points_transformer, hits_transformer=hits_transformer)\n", "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", " size=10, fill_alpha=0)\n", "\n", @@ -296,7 +313,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We could view the result above by uncommenting the last line, but let's just go ahead and make a little app with widgets for controlling the visibility of the background map, the data, and the text labels, plus a table showing information about the selected vessel:" + "We could view the result above by uncommenting the last line, but let's just go ahead and make a little [Panel](https://panel.holoviz.org) app so we can add a few extra interactive features. First, some code to fetch a photo of the selected vessel, if available:" ] }, { @@ -307,10 +324,9 @@ "source": [ "def get_photo_url(mmsi):\n", " headers = {'User-Agent': 'Mozilla/5.0'}\n", - " r=requests.get(f'https://www.marinetraffic.com/cs/ais/details/ships/mmsi:{mmsi}', \n", - " allow_redirects=True, headers=headers)\n", + " r=requests.get(f'{ship_url}{mmsi}', allow_redirects=True, headers=headers)\n", " ship_id = [el for el in r.url.split('/') if el.startswith('shipid')]\n", - " if ship_id == []: return 'Not found'\n", + " if ship_id == []: return ''\n", " ship_id =ship_id[0].replace('shipid:','')\n", " return f\"https://photos.marinetraffic.com/ais/showphoto.aspx?shipid={ship_id}&size=thumb300&stamp=false\"\n", "\n", @@ -321,40 +337,68 @@ " im = Image.open(response.raw)\n", " except: \n", " im = Image.new('RGB', (1,1), (255, 255, 255))\n", - " return pn.panel(im)\n" + " return pn.panel(im)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we can build an app with some widgets for controlling the visibility of the background map, the data, and the text labels, plus a table showing information about the selected vessel and a photo if available:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ - "photo = pn.bind(get_photo, hits=highlighter.param.hits)\n", - "\n", - "map_opacity = pn.widgets.FloatSlider(start=0, end=1, value=0.7, name=\"Map opacity\")\n", - "data_opacity = pn.widgets.FloatSlider(start=0, end=1, value=1.0, name=\"Data opacity\")\n", - "label_opacity = pn.widgets.FloatSlider(start=0, end=1, value=0.9, name=\"Label opacity\")\n", - "\n", - "overlay = (tiles.apply.opts(alpha=map_opacity) *\n", - " pointsp.apply.opts(alpha=data_opacity) *\n", - " labels.apply.opts(alpha=label_opacity) * highlight * legend)\n", - "table = pn.bind(pn.pane.DataFrame, object=highlighter.param.hits, \n", - " width=1200, index=False, render_links=True, na_rep='')\n", + "photo = pn.bind(get_photo, hits=highlighter.param.hits)\n", + "sopts = dict(start=0, end=1, sizing_mode='stretch_width')\n", + "map_opacity = pn.widgets.FloatSlider(value=0.7, name=\"Map opacity\", **sopts)\n", + "data_opacity = pn.widgets.FloatSlider(value=1.0, name=\"Data opacity\", **sopts)\n", + "label_opacity = pn.widgets.FloatSlider(value=0.9, name=\"Label opacity\", **sopts)\n", + "overlay = (tiles.apply.opts(alpha=map_opacity) *\n", + " pointsp.apply.opts(alpha=data_opacity) *\n", + " labels.apply.opts(alpha=label_opacity) * highlight * legend)\n", + "table = pn.bind(pn.pane.DataFrame, object=highlighter.param.hits, \n", + " index=False, render_links=True, na_rep='', sizing_mode='stretch_width')\n", "\n", "pn.Column(\"# Categorical plot of AIS data by type\",\n", " \"Zoom or pan to explore the data, then click to select \"\n", " \"a particular data point to see more information about it (after a delay). \"\n", " \"You may need to zoom in before a point is selectable.\",\n", " pn.Row(map_opacity, data_opacity, label_opacity),\n", - " overlay, table, photo, sizing_mode='stretch_width').servable()" + " overlay, table, photo, sizing_mode='stretch_width').servable()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This app should run fine in a Jupyter notebook, but it can also be launched as a separate web server using `panel serve --port 5006 ship_traffic.ipynb`, allowing you to let other people explore this dataset as well." ] } ], "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "pygments_lexer": "ipython3" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.11" } }, "nbformat": 4, From d175360b84674101acbd4ea167f9333eaf3f5378 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Sun, 24 Jan 2021 14:14:25 -0600 Subject: [PATCH 26/46] Simplified AIS category descriptions --- ship_traffic/AIS_categories.csv | 374 ++++++++++++++++---------------- 1 file changed, 187 insertions(+), 187 deletions(-) diff --git a/ship_traffic/AIS_categories.csv b/ship_traffic/AIS_categories.csv index 630270c4a..7f43e2e7d 100644 --- a/ship_traffic/AIS_categories.csv +++ b/ship_traffic/AIS_categories.csv @@ -19,16 +19,16 @@ num,desc,category,category_desc 17,Reserved,0,Unknown 18,Reserved,0,Unknown 19,Reserved,0,Unknown -20,"Wing in ground (WIG), all ships of this type",1,WIG +20,"Wing in ground (WIG)",1,WIG 21,"Wing in ground (WIG), Hazardous category A",1,WIG 22,"Wing in ground (WIG), Hazardous category B",1,WIG 23,"Wing in ground (WIG), Hazardous category C",1,WIG 24,"Wing in ground (WIG), Hazardous category D",1,WIG -25,"Wing in ground (WIG), Reserved for future use",1,WIG -26,"Wing in ground (WIG), Reserved for future use",1,WIG -27,"Wing in ground (WIG), Reserved for future use",1,WIG -28,"Wing in ground (WIG), Reserved for future use",1,WIG -29,"Wing in ground (WIG), Reserved for future use",1,WIG +25,"Wing in ground (WIG), Reserved",1,WIG +26,"Wing in ground (WIG), Reserved",1,WIG +27,"Wing in ground (WIG), Reserved",1,WIG +28,"Wing in ground (WIG), Reserved",1,WIG +29,"Wing in ground (WIG), Reserved",1,WIG 30,Fishing,2,Fishing 31,Towing,3,Towing 32,Towing: length exceeds 200m or breadth exceeds 25m,3,Towing @@ -39,15 +39,15 @@ num,desc,category,category_desc 37,Pleasure Craft,8,Pleasure 38,Reserved,0,Unknown 39,Reserved,0,Unknown -40,"High speed craft (HSC), all ships of this type",9,High Speed +40,"High speed craft (HSC)",9,High Speed 41,"High speed craft (HSC), Hazardous category A",9,High Speed 42,"High speed craft (HSC), Hazardous category B",9,High Speed 43,"High speed craft (HSC), Hazardous category C",9,High Speed 44,"High speed craft (HSC), Hazardous category D",9,High Speed -45,"High speed craft (HSC), Reserved for future use",9,High Speed -46,"High speed craft (HSC), Reserved for future use",9,High Speed -47,"High speed craft (HSC), Reserved for future use",9,High Speed -48,"High speed craft (HSC), Reserved for future use",9,High Speed +45,"High speed craft (HSC), Reserved",9,High Speed +46,"High speed craft (HSC), Reserved",9,High Speed +47,"High speed craft (HSC), Reserved",9,High Speed +48,"High speed craft (HSC), Reserved",9,High Speed 49,"High speed craft (HSC), No additional information",9,High Speed 50,Pilot Vessel,10,Pilot Vessel 51,Search and Rescue vessel,11,Search and Rescue vessel @@ -59,202 +59,202 @@ num,desc,category,category_desc 57,Spare - Local Vessel,15,Spare 58,Medical Transport,16,Medical Transport 59,Noncombatant ship according to RR Resolution No. 18,17,Noncombatant -60,"Passenger, all ships of this type",18,Passenger +60,"Passenger",18,Passenger 61,"Passenger, Hazardous category A",18,Passenger 62,"Passenger, Hazardous category B",18,Passenger 63,"Passenger, Hazardous category C",18,Passenger 64,"Passenger, Hazardous category D",18,Passenger -65,"Passenger, Reserved for future use",18,Passenger -66,"Passenger, Reserved for future use",18,Passenger -67,"Passenger, Reserved for future use",18,Passenger -68,"Passenger, Reserved for future use",18,Passenger +65,"Passenger, Reserved",18,Passenger +66,"Passenger, Reserved",18,Passenger +67,"Passenger, Reserved",18,Passenger +68,"Passenger, Reserved",18,Passenger 69,"Passenger, No additional information",18,Passenger -70,"Cargo, all ships of this type",19,Cargo +70,"Cargo",19,Cargo 71,"Cargo, Hazardous category A",19,Cargo 72,"Cargo, Hazardous category B",19,Cargo 73,"Cargo, Hazardous category C",19,Cargo 74,"Cargo, Hazardous category D",19,Cargo -75,"Cargo, Reserved for future use",19,Cargo -76,"Cargo, Reserved for future use",19,Cargo -77,"Cargo, Reserved for future use",19,Cargo -78,"Cargo, Reserved for future use",19,Cargo +75,"Cargo, Reserved",19,Cargo +76,"Cargo, Reserved",19,Cargo +77,"Cargo, Reserved",19,Cargo +78,"Cargo, Reserved",19,Cargo 79,"Cargo, No additional information",19,Cargo -80,"Tanker, all ships of this type",20,Tanker +80,"Tanker",20,Tanker 81,"Tanker, Hazardous category A",20,Tanker 82,"Tanker, Hazardous category B",20,Tanker 83,"Tanker, Hazardous category C",20,Tanker 84,"Tanker, Hazardous category D",20,Tanker -85,"Tanker, Reserved for future use",20,Tanker -86,"Tanker, Reserved for future use",20,Tanker -87,"Tanker, Reserved for future use",20,Tanker -88,"Tanker, Reserved for future use",20,Tanker +85,"Tanker, Reserved",20,Tanker +86,"Tanker, Reserved",20,Tanker +87,"Tanker, Reserved",20,Tanker +88,"Tanker, Reserved",20,Tanker 89,"Tanker, No additional information",20,Tanker -90,"Other Type, all ships of this type",21,Other +90,"Other Type",21,Other 91,"Other Type, Hazardous category A",21,Other 92,"Other Type, Hazardous category B",21,Other 93,"Other Type, Hazardous category C",21,Other 94,"Other Type, Hazardous category D",21,Other -95,"Other Type, Reserved for future use",21,Other -96,"Other Type, Reserved for future use",21,Other -97,"Other Type, Reserved for future use",21,Other -98,"Other Type, Reserved for future use",21,Other +95,"Other Type, Reserved",21,Other +96,"Other Type, Reserved",21,Other +97,"Other Type, Reserved",21,Other +98,"Other Type, Reserved",21,Other 99,"Other Type, no additional information",21,Other -100,Reserved for future use,0,Unknown -101,Reserved for future use,0,Unknown -102,Reserved for future use,0,Unknown -103,Reserved for future use,0,Unknown -104,Reserved for future use,0,Unknown -105,Reserved for future use,0,Unknown -106,Reserved for future use,0,Unknown -107,Reserved for future use,0,Unknown -108,Reserved for future use,0,Unknown -109,Reserved for future use,0,Unknown -110,Reserved for future use,0,Unknown -111,Reserved for future use,0,Unknown -112,Reserved for future use,0,Unknown -113,Reserved for future use,0,Unknown -114,Reserved for future use,0,Unknown -115,Reserved for future use,0,Unknown -116,Reserved for future use,0,Unknown -117,Reserved for future use,0,Unknown -118,Reserved for future use,0,Unknown -119,Reserved for future use,0,Unknown -120,Reserved for future use,0,Unknown -121,Reserved for future use,0,Unknown -122,Reserved for future use,0,Unknown -123,Reserved for future use,0,Unknown -124,Reserved for future use,0,Unknown -125,Reserved for future use,0,Unknown -126,Reserved for future use,0,Unknown -127,Reserved for future use,0,Unknown -128,Reserved for future use,0,Unknown -129,Reserved for future use,0,Unknown -130,Reserved for future use,0,Unknown -131,Reserved for future use,0,Unknown -132,Reserved for future use,0,Unknown -133,Reserved for future use,0,Unknown -134,Reserved for future use,0,Unknown -135,Reserved for future use,0,Unknown -136,Reserved for future use,0,Unknown -137,Reserved for future use,0,Unknown -138,Reserved for future use,0,Unknown -139,Reserved for future use,0,Unknown -140,Reserved for future use,0,Unknown -141,Reserved for future use,0,Unknown -142,Reserved for future use,0,Unknown -143,Reserved for future use,0,Unknown -144,Reserved for future use,0,Unknown -145,Reserved for future use,0,Unknown -146,Reserved for future use,0,Unknown -147,Reserved for future use,0,Unknown -148,Reserved for future use,0,Unknown -149,Reserved for future use,0,Unknown -150,Reserved for future use,0,Unknown -151,Reserved for future use,0,Unknown -152,Reserved for future use,0,Unknown -153,Reserved for future use,0,Unknown -154,Reserved for future use,0,Unknown -155,Reserved for future use,0,Unknown -156,Reserved for future use,0,Unknown -157,Reserved for future use,0,Unknown -158,Reserved for future use,0,Unknown -159,Reserved for future use,0,Unknown -160,Reserved for future use,0,Unknown -161,Reserved for future use,0,Unknown -162,Reserved for future use,0,Unknown -163,Reserved for future use,0,Unknown -164,Reserved for future use,0,Unknown -165,Reserved for future use,0,Unknown -166,Reserved for future use,0,Unknown -167,Reserved for future use,0,Unknown -168,Reserved for future use,0,Unknown -169,Reserved for future use,0,Unknown -170,Reserved for future use,0,Unknown -171,Reserved for future use,0,Unknown -172,Reserved for future use,0,Unknown -173,Reserved for future use,0,Unknown -174,Reserved for future use,0,Unknown -175,Reserved for future use,0,Unknown -176,Reserved for future use,0,Unknown -177,Reserved for future use,0,Unknown -178,Reserved for future use,0,Unknown -179,Reserved for future use,0,Unknown -180,Reserved for future use,0,Unknown -181,Reserved for future use,0,Unknown -182,Reserved for future use,0,Unknown -183,Reserved for future use,0,Unknown -184,Reserved for future use,0,Unknown -185,Reserved for future use,0,Unknown -186,Reserved for future use,0,Unknown -187,Reserved for future use,0,Unknown -188,Reserved for future use,0,Unknown -189,Reserved for future use,0,Unknown -190,Reserved for future use,0,Unknown -191,Reserved for future use,0,Unknown -192,Reserved for future use,0,Unknown -193,Reserved for future use,0,Unknown -194,Reserved for future use,0,Unknown -195,Reserved for future use,0,Unknown -196,Reserved for future use,0,Unknown -197,Reserved for future use,0,Unknown -198,Reserved for future use,0,Unknown -199,Reserved for future use,0,Unknown -200,Reserved for future use,0,Unknown -201,Reserved for future use,0,Unknown -202,Reserved for future use,0,Unknown -203,Reserved for future use,0,Unknown -204,Reserved for future use,0,Unknown -205,Reserved for future use,0,Unknown -206,Reserved for future use,0,Unknown -207,Reserved for future use,0,Unknown -208,Reserved for future use,0,Unknown -209,Reserved for future use,0,Unknown -210,Reserved for future use,0,Unknown -211,Reserved for future use,0,Unknown -212,Reserved for future use,0,Unknown -213,Reserved for future use,0,Unknown -214,Reserved for future use,0,Unknown -215,Reserved for future use,0,Unknown -216,Reserved for future use,0,Unknown -217,Reserved for future use,0,Unknown -218,Reserved for future use,0,Unknown -219,Reserved for future use,0,Unknown -220,Reserved for future use,0,Unknown -221,Reserved for future use,0,Unknown -222,Reserved for future use,0,Unknown -223,Reserved for future use,0,Unknown -224,Reserved for future use,0,Unknown -225,Reserved for future use,0,Unknown -226,Reserved for future use,0,Unknown -227,Reserved for future use,0,Unknown -228,Reserved for future use,0,Unknown -229,Reserved for future use,0,Unknown -230,Reserved for future use,0,Unknown -231,Reserved for future use,0,Unknown -232,Reserved for future use,0,Unknown -233,Reserved for future use,0,Unknown -234,Reserved for future use,0,Unknown -235,Reserved for future use,0,Unknown -236,Reserved for future use,0,Unknown -237,Reserved for future use,0,Unknown -238,Reserved for future use,0,Unknown -239,Reserved for future use,0,Unknown -240,Reserved for future use,0,Unknown -241,Reserved for future use,0,Unknown -242,Reserved for future use,0,Unknown -243,Reserved for future use,0,Unknown -244,Reserved for future use,0,Unknown -245,Reserved for future use,0,Unknown -246,Reserved for future use,0,Unknown -247,Reserved for future use,0,Unknown -248,Reserved for future use,0,Unknown -249,Reserved for future use,0,Unknown -250,Reserved for future use,0,Unknown -251,Reserved for future use,0,Unknown -252,Reserved for future use,0,Unknown -253,Reserved for future use,0,Unknown -254,Reserved for future use,0,Unknown -255,Reserved for future use,0,Unknown +100,Reserved,0,Unknown +101,Reserved,0,Unknown +102,Reserved,0,Unknown +103,Reserved,0,Unknown +104,Reserved,0,Unknown +105,Reserved,0,Unknown +106,Reserved,0,Unknown +107,Reserved,0,Unknown +108,Reserved,0,Unknown +109,Reserved,0,Unknown +110,Reserved,0,Unknown +111,Reserved,0,Unknown +112,Reserved,0,Unknown +113,Reserved,0,Unknown +114,Reserved,0,Unknown +115,Reserved,0,Unknown +116,Reserved,0,Unknown +117,Reserved,0,Unknown +118,Reserved,0,Unknown +119,Reserved,0,Unknown +120,Reserved,0,Unknown +121,Reserved,0,Unknown +122,Reserved,0,Unknown +123,Reserved,0,Unknown +124,Reserved,0,Unknown +125,Reserved,0,Unknown +126,Reserved,0,Unknown +127,Reserved,0,Unknown +128,Reserved,0,Unknown +129,Reserved,0,Unknown +130,Reserved,0,Unknown +131,Reserved,0,Unknown +132,Reserved,0,Unknown +133,Reserved,0,Unknown +134,Reserved,0,Unknown +135,Reserved,0,Unknown +136,Reserved,0,Unknown +137,Reserved,0,Unknown +138,Reserved,0,Unknown +139,Reserved,0,Unknown +140,Reserved,0,Unknown +141,Reserved,0,Unknown +142,Reserved,0,Unknown +143,Reserved,0,Unknown +144,Reserved,0,Unknown +145,Reserved,0,Unknown +146,Reserved,0,Unknown +147,Reserved,0,Unknown +148,Reserved,0,Unknown +149,Reserved,0,Unknown +150,Reserved,0,Unknown +151,Reserved,0,Unknown +152,Reserved,0,Unknown +153,Reserved,0,Unknown +154,Reserved,0,Unknown +155,Reserved,0,Unknown +156,Reserved,0,Unknown +157,Reserved,0,Unknown +158,Reserved,0,Unknown +159,Reserved,0,Unknown +160,Reserved,0,Unknown +161,Reserved,0,Unknown +162,Reserved,0,Unknown +163,Reserved,0,Unknown +164,Reserved,0,Unknown +165,Reserved,0,Unknown +166,Reserved,0,Unknown +167,Reserved,0,Unknown +168,Reserved,0,Unknown +169,Reserved,0,Unknown +170,Reserved,0,Unknown +171,Reserved,0,Unknown +172,Reserved,0,Unknown +173,Reserved,0,Unknown +174,Reserved,0,Unknown +175,Reserved,0,Unknown +176,Reserved,0,Unknown +177,Reserved,0,Unknown +178,Reserved,0,Unknown +179,Reserved,0,Unknown +180,Reserved,0,Unknown +181,Reserved,0,Unknown +182,Reserved,0,Unknown +183,Reserved,0,Unknown +184,Reserved,0,Unknown +185,Reserved,0,Unknown +186,Reserved,0,Unknown +187,Reserved,0,Unknown +188,Reserved,0,Unknown +189,Reserved,0,Unknown +190,Reserved,0,Unknown +191,Reserved,0,Unknown +192,Reserved,0,Unknown +193,Reserved,0,Unknown +194,Reserved,0,Unknown +195,Reserved,0,Unknown +196,Reserved,0,Unknown +197,Reserved,0,Unknown +198,Reserved,0,Unknown +199,Reserved,0,Unknown +200,Reserved,0,Unknown +201,Reserved,0,Unknown +202,Reserved,0,Unknown +203,Reserved,0,Unknown +204,Reserved,0,Unknown +205,Reserved,0,Unknown +206,Reserved,0,Unknown +207,Reserved,0,Unknown +208,Reserved,0,Unknown +209,Reserved,0,Unknown +210,Reserved,0,Unknown +211,Reserved,0,Unknown +212,Reserved,0,Unknown +213,Reserved,0,Unknown +214,Reserved,0,Unknown +215,Reserved,0,Unknown +216,Reserved,0,Unknown +217,Reserved,0,Unknown +218,Reserved,0,Unknown +219,Reserved,0,Unknown +220,Reserved,0,Unknown +221,Reserved,0,Unknown +222,Reserved,0,Unknown +223,Reserved,0,Unknown +224,Reserved,0,Unknown +225,Reserved,0,Unknown +226,Reserved,0,Unknown +227,Reserved,0,Unknown +228,Reserved,0,Unknown +229,Reserved,0,Unknown +230,Reserved,0,Unknown +231,Reserved,0,Unknown +232,Reserved,0,Unknown +233,Reserved,0,Unknown +234,Reserved,0,Unknown +235,Reserved,0,Unknown +236,Reserved,0,Unknown +237,Reserved,0,Unknown +238,Reserved,0,Unknown +239,Reserved,0,Unknown +240,Reserved,0,Unknown +241,Reserved,0,Unknown +242,Reserved,0,Unknown +243,Reserved,0,Unknown +244,Reserved,0,Unknown +245,Reserved,0,Unknown +246,Reserved,0,Unknown +247,Reserved,0,Unknown +248,Reserved,0,Unknown +249,Reserved,0,Unknown +250,Reserved,0,Unknown +251,Reserved,0,Unknown +252,Reserved,0,Unknown +253,Reserved,0,Unknown +254,Reserved,0,Unknown +255,Reserved,0,Unknown 1001,Fishing vessels,2,Fishing 1002,Fishing vessels,2,Fishing 1003,Freight Vessels,19,Cargo From 87d1ec0f22f76d0b24f9f35c540865ad08e194bc Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Sun, 24 Jan 2021 14:14:25 -0600 Subject: [PATCH 27/46] Updated .yml --- ship_traffic/anaconda-project.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ship_traffic/anaconda-project.yml b/ship_traffic/anaconda-project.yml index 972c7cdca..2a3db7bcf 100644 --- a/ship_traffic/anaconda-project.yml +++ b/ship_traffic/anaconda-project.yml @@ -1,6 +1,6 @@ # To reproduce: install 'anaconda-project', then 'anaconda-project run' -name: ship_tracking -description: Visualizing AIS tracking data for ships near the USA +name: ship_traffic +description: Visualizing AIS location tracking data for marine vessels near the USA maintainers: - jbednar labels: @@ -33,10 +33,10 @@ dependencies: *pkgs commands: dashboard: - unix: panel serve ship_tracking.ipynb + unix: panel serve ship_traffic.ipynb supports_http_options: true notebook: - notebook: ship_tracking.ipynb + notebook: ship_traffic.ipynb test: unix: pytest --nbsmoke-run -k *.ipynb --ignore envs windows: pytest --nbsmoke-run -k *.ipynb --ignore envs From d384e9f91e6436aacc7da47b1cfb1f62ebfaad8f Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Sun, 24 Jan 2021 14:19:15 -0600 Subject: [PATCH 28/46] Simplified hit and photo handling --- ship_traffic/ship_traffic.ipynb | 85 +++++++++++++++------------------ 1 file changed, 38 insertions(+), 47 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 75e013ad2..cc9af7039 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -114,9 +114,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Load AIS pings\n", + "## Load traffic data\n", "\n", - "Next we will load the data from disk, either directly from a spatially indexed Parquet file (if previously cached) or from the raw CSV files. We'll also project the data to the coordinate system we will use later for plotting. There's a lot of data to process, so we'll use [Dask](https://dask.org) to ensure that we use all the cores available on this machine. Dask breaks a dataset into partitions that can be processed in parallel, so here we define a function for dealing with one partition's worth of data, along with a schema showing what the final dataframe's columnar structure will be." + "Next we will load the data from disk, either directly from a spatially indexed Parquet file (if previously cached) or from the raw CSV files. We'll also project the location data to the coordinate system we will use later for plotting. There's a lot of data to process, so we'll use [Dask](https://dask.org) to ensure that we use all the cores available on this machine. Dask breaks a dataset into partitions that can be processed in parallel, so here we define a function for dealing with one partition's worth of data, along with a schema showing what the final dataframe's columnar structure will be." ] }, { @@ -185,9 +185,9 @@ " gdf = gdf.pack_partitions_to_parquet(cache_file, npartitions=64).persist()\n", " \n", " with ProgressBar(): \n", - " print('Building spatial index') # Takes a couple of minutes for 1 month's data\n", - " if spatial_index: gdf = gdf.build_sindex()\n", - " gdf = gdf.persist()\n", + " if spatial_index: \n", + " print('Building spatial index') # Takes a couple of minutes for 1 month's data\n", + " gdf = gdf.build_sindex().persist()\n", " gdf['category'] = gdf['category'].astype('category').cat.as_known()\n", "\n", " return gdf, vessels" @@ -197,7 +197,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now let's actually load the data, using the disk cache and memory cache if available. You can change `spatial_index` to `True` above to speed up selection of individual points in the final plot, though `load_data` will then take several minutes rather than a few seconds." + "Now let's actually load the data, using the disk cache and memory cache if available. If you set `spatial_index` to `True` above it should speed up selection of individual points in the final plot, though `load_data` will then take several minutes rather than a few seconds." ] }, { @@ -275,20 +275,21 @@ "vessels_df = vessels.compute()\n", "columns = ['VesselName', 'MMSI', 'IMO', 'VesselType']\n", "\n", - "def points_transformer(df):\n", - " return df.merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]\n", + "def format_vessel_type(num):\n", + " if np.isnan(num): num = 0\n", + " return f'{num:.0f} ({vessel_types.loc[num].desc})'\n", + "\n", + "def partial_vessel_record(df):\n", + " return df.iloc[:1].merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]\n", "\n", "ship_url = 'https://tinyurl.com/aispage/mmsi:'\n", - "def hits_transformer(df):\n", - " match = pd.DataFrame(points_transformer(df)[columns].iloc[0:1])\n", - " url = vdesc = ''\n", - " if len(match)==1:\n", - " row = match.iloc[0]\n", - " url = f'{ship_url}{row.MMSI}'\n", - " vdesc = f'{row.VesselType:.0f} ({vessel_types.loc[row.VesselType].desc})'\n", - " match['URL'] = pd.Series([url])\n", - " match['VesselType'] = pd.Series([vdesc])\n", - " return match" + "def full_vessel_record(df, records_to_return=2):\n", + " \"Given a dataframe that includes MMSI values, return an augmented dataframe with URL and vessel info included\"\n", + " df_with_info = df.iloc[:records_to_return].merge(vessels_df, on='MMSI')\n", + " df_with_types = df_with_info.merge(vessel_types, how='left', left_on='VesselType', right_on='num')[columns]\n", + " df_with_types['URL'] = df_with_types.MMSI.apply(lambda x: f'{ship_url}{x}')\n", + " df_with_types.VesselType = df_with_types.VesselType.apply(format_vessel_type)\n", + " return pd.DataFrame(df_with_types).drop_duplicates()" ] }, { @@ -298,11 +299,12 @@ "outputs": [], "source": [ "xr, yr = ll2en([-126,-120.7], [47.5,49.5])\n", + "#xr, yr = ll2en([-78.2, -74.5], [40.6,42.0])\n", "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", "pointsp = hd.dynspread(hd.datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], \n", - " points_transformer=points_transformer, hits_transformer=hits_transformer)\n", + " points_transformer=partial_vessel_record, hits_transformer=full_vessel_record)\n", "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", " size=10, fill_alpha=0)\n", "\n", @@ -313,7 +315,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We could view the result above by uncommenting the last line, but let's just go ahead and make a little [Panel](https://panel.holoviz.org) app so we can add a few extra interactive features. First, some code to fetch a photo of the selected vessel, if available:" + "We could view the result above by uncommenting the last line, but let's just go ahead and make a little [Panel](https://panel.holoviz.org) app so we can add a few extra interactive features. First, some code to fetch a photo of the selected vessels, if available:" ] }, { @@ -330,14 +332,18 @@ " ship_id =ship_id[0].replace('shipid:','')\n", " return f\"https://photos.marinetraffic.com/ais/showphoto.aspx?shipid={ship_id}&size=thumb300&stamp=false\"\n", "\n", - "def get_photo(hits=None):\n", - " try:\n", - " url = get_photo_url(hits.iloc[0]['MMSI'])\n", - " response = requests.get(url, stream=True)\n", - " im = Image.open(response.raw)\n", - " except: \n", - " im = Image.new('RGB', (1,1), (255, 255, 255))\n", - " return pn.panel(im)" + "def get_photos(df=None):\n", + " photos = []\n", + " if df is not None and 'MMSI' in df.columns:\n", + " for mmsi in df.MMSI.to_list():\n", + " try:\n", + " url = get_photo_url(mmsi)\n", + " response = requests.get(url, stream=True)\n", + " im = Image.open(response.raw)\n", + " photos += [pn.Column(mmsi,im)] \n", + " except:\n", + " pass\n", + " return pn.Row(*photos)" ] }, { @@ -350,12 +356,10 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ - "photo = pn.bind(get_photo, hits=highlighter.param.hits)\n", + "photos = pn.bind(get_photos, df=highlighter.param.hits)\n", "sopts = dict(start=0, end=1, sizing_mode='stretch_width')\n", "map_opacity = pn.widgets.FloatSlider(value=0.7, name=\"Map opacity\", **sopts)\n", "data_opacity = pn.widgets.FloatSlider(value=1.0, name=\"Data opacity\", **sopts)\n", @@ -371,7 +375,7 @@ " \"a particular data point to see more information about it (after a delay). \"\n", " \"You may need to zoom in before a point is selectable.\",\n", " pn.Row(map_opacity, data_opacity, label_opacity),\n", - " overlay, table, photo, sizing_mode='stretch_width').servable()" + " overlay, table, photos, sizing_mode='stretch_width').servable()" ] }, { @@ -383,22 +387,9 @@ } ], "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.11" + "pygments_lexer": "ipython3" } }, "nbformat": 4, From 4bc27b256f4a9ed97105b9690c07229f92aa52c2 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Sun, 24 Jan 2021 14:53:45 -0600 Subject: [PATCH 29/46] Made .yml whitespace match anaconda-project auto-generated formatting --- ship_traffic/anaconda-project.yml | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ship_traffic/anaconda-project.yml b/ship_traffic/anaconda-project.yml index 2a3db7bcf..e652e600d 100644 --- a/ship_traffic/anaconda-project.yml +++ b/ship_traffic/anaconda-project.yml @@ -10,24 +10,24 @@ labels: user_fields: [labels, skip, maintainers] channels: - - pyviz +- pyviz packages: &pkgs - - bokeh ==2.2.3 - - colorcet ==2 - - dask ==2020.12.0 - - datashader ==0.12.0 - - holoviews ==1.14.0 - - notebook ==6.1.5 - - numba ==0.51.2 - - numexpr ==2.7.1 - - pandas ==1.1.5 - - panel ==0.10.3 - - python ==3.7.9 - - spatialpandas ==0.3.6 - - xarray ==0.16.2 - - pip ==20.3.3 - - conda-forge::pyarrow ==2 +- bokeh ==2.2.3 +- colorcet ==2 +- dask ==2020.12.0 +- datashader ==0.12.0 +- holoviews ==1.14.0 +- notebook ==6.1.5 +- numba ==0.51.2 +- numexpr ==2.7.1 +- pandas ==1.1.5 +- panel ==0.10.3 +- python ==3.7.9 +- spatialpandas ==0.3.6 +- xarray ==0.16.2 +- pip ==20.3.3 +- conda-forge::pyarrow ==2 dependencies: *pkgs From c6fb0137ec5b8e64e5bbcf4205e414d5d7beaea6 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Sun, 24 Jan 2021 14:58:30 -0600 Subject: [PATCH 30/46] Cleaned up title formatting --- ship_traffic/ship_traffic.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index cc9af7039..f20ee256f 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -370,7 +370,7 @@ "table = pn.bind(pn.pane.DataFrame, object=highlighter.param.hits, \n", " index=False, render_links=True, na_rep='', sizing_mode='stretch_width')\n", "\n", - "pn.Column(\"# Categorical plot of AIS data by type\",\n", + "pn.Column(\"## US AIS vessel traffic data, Jan 2020\\n\"\n", " \"Zoom or pan to explore the data, then click to select \"\n", " \"a particular data point to see more information about it (after a delay). \"\n", " \"You may need to zoom in before a point is selectable.\",\n", From f2870db305f355b36e3b15b766bbe05d8ca11caa Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Sun, 24 Jan 2021 15:21:37 -0600 Subject: [PATCH 31/46] Added max_hits slider --- ship_traffic/ship_traffic.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index f20ee256f..acc031d3c 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -303,8 +303,10 @@ "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", "pointsp = hd.dynspread(hd.datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", + "max_hits = pn.widgets.IntSlider(value=2, start=1, end=10, name=\"Max hits\", sizing_mode='stretch_width')\n", + "\n", "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], \n", - " points_transformer=partial_vessel_record, hits_transformer=full_vessel_record)\n", + " points_transformer=partial_vessel_record, hits_transformer=pn.bind(full_vessel_record, records_to_return=max_hits))\n", "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", " size=10, fill_alpha=0)\n", "\n", @@ -374,7 +376,7 @@ " \"Zoom or pan to explore the data, then click to select \"\n", " \"a particular data point to see more information about it (after a delay). \"\n", " \"You may need to zoom in before a point is selectable.\",\n", - " pn.Row(map_opacity, data_opacity, label_opacity),\n", + " pn.Row(map_opacity, data_opacity, label_opacity, max_hits),\n", " overlay, table, photos, sizing_mode='stretch_width').servable()" ] }, From d8cd5736fe026064bc97f40abc8cc031907d554f Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Sun, 24 Jan 2021 20:54:33 -0600 Subject: [PATCH 32/46] Added static plots of predefined locations --- ship_traffic/ship_traffic.ipynb | 118 ++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 12 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index acc031d3c..234b99833 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -24,7 +24,7 @@ "from holoviews.util.transform import lon_lat_to_easting_northing as ll2en\n", "from dask.diagnostics import ProgressBar\n", "\n", - "hv.extension('bokeh', width=100)" + "hv.extension('bokeh', 'matplotlib', width=100)" ] }, { @@ -230,11 +230,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Plot categorical data\n", + "## Predefined locations\n", "\n", - "We can now plot the data colored by category, with a color key.\n", - "\n", - "To zoom in & interact with the plot, click the “Wheel zoom” tool in the [Bokeh toolbar](https://docs.bokeh.org/en/latest/docs/user_guide/tools.html) on the side of the plot and click and drag the plot in order to look around, or use the \"Box Zoom\" tool to select an area of interest. As you zoom in, finer-grained detail will emerge and fill in, as long as you have a live Python process running to render the data dynamically. Depending on the size of the dataset and your machine, updating the plot might take a few seconds." + "We'll provide interactive plots later that let you zoom in anywhere you like, but first let's highlight a few specific areas of interest for those without a live Python process to interact with:" ] }, { @@ -243,10 +241,104 @@ "metadata": {}, "outputs": [], "source": [ + "def ranges(lon_range, lat_range):\n", + " x_range, y_range = ll2en(lon_range, lat_range)\n", + " return dict(x=tuple(x_range), y=tuple(y_range))\n", + "\n", "x_range, y_range = ll2en([-54,-132], [15,51])\n", "bounds = dict(x=tuple(x_range), y=tuple(y_range))\n", "\n", - "pts = hv.Points(df, vdims=['category']).redim.range(**bounds)\n", + "loc = {\n", + " 'Continental US': ranges((-132.0, -54.0), (15.0, 51.0)),\n", + " 'Vancouver Area': ranges((-126.0, -120.7), (47.5, 49.5)),\n", + " 'NY and NJ': ranges(( -75.6, -71.3), (39.4, 41.1)),\n", + " 'Gulf of Mexico': ranges(( -98.0, -81.0), (23.8, 32.0)),\n", + " 'Gulf Coast': ranges(( -98.0, -87.0), (25.2, 31.0)),\n", + " 'Louisiana Coast': ranges(( -91.5, -87.8), (28.4, 30.1)),\n", + " 'Mississipi Delta': ranges(( -90.1, -89.2), (28.65,29.15)),\n", + " 'Louisiana East Bay': ranges(( -89.37, -89.28),(28.86,28.9)),\n", + " 'Bering Sea': ranges((-171.0, -159.0), (52.0, 56.0)),\n", + " 'Hawaii': ranges((-160.0, -154.5), (19.5, 22.1)),\n", + " 'LA to San Diego': ranges((-120.5, -117.0), (32.6, 34.1)),\n", + " 'Great Lakes': ranges(( -89.0, -77.0), (41.2, 46.1)),\n", + " 'Chesapeake Bay': ranges(( -78.0, -71.0), (36.4, 39.6)),\n", + " 'Pamlico Sound, NC': ranges(( -80.0, -72.5), (33.1, 36.8)),\n", + " 'Savannah, GA': ranges(( -81.2, -80.3), (31.85,32.25)),\n", + " 'Florida': ranges(( -90.0, -74.5), (23.3, 31.0)),\n", + " 'Puerto Rico': ranges(( -68.1, -64.2), (17.4, 19.5))}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can easily render these to PNGs using HoloViews to call Datashader and render the results using Matplotlib:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hv.output(backend='matplotlib')\n", + "hv.opts.defaults(\n", + " hv.opts.RGB(xaxis=None, yaxis=None, axiswise=True, bgcolor='black'),\n", + " hv.opts.Layout(hspace=0.0, vspace=0.1, sublabel_format=None, framewise=True, fig_size=400))\n", + "\n", + "plots = [hd.datashade(hv.Points(df), color_key=color_key, cmap=cc.fire, width=1000, height=600,\n", + " dynamic=False, x_range=ranges['x'], y_range=ranges['y']).relabel(region)\n", + " for region, ranges in loc.items()]\n", + "hv.Layout(plots).cols(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Even more structure is visible if we color by the vessel category (using the color key shown in later Bokeh plots):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plots = [hd.datashade(hv.Points(df, vdims='category'), color_key=color_key,\n", + " aggregator=ds.count_cat('category'), width=1000, height=600,\n", + " dynamic=False, x_range=ranges['x'], y_range=ranges['y']).relabel(region)\n", + " for region, ranges in loc.items()]\n", + "hv.Layout(plots).cols(1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hv.output(backend='bokeh')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Interactive plots\n", + "\n", + "To let you explore the data yourself, we can plot it using the [Bokeh](https://bokeh.org) backend, which provides JavaScript-based interactive plotting in a web browser. \n", + "\n", + "To zoom in & interact with the plot, click the “Wheel zoom” tool in the [Bokeh toolbar](https://docs.bokeh.org/en/latest/docs/user_guide/tools.html) on the side of the plot and click and drag the plot in order to look around, or use the \"Box Zoom\" tool to select an area of interest. As you zoom in, finer-grained detail will emerge and fill in, as long as you have a live Python process running to render the data dynamically using Datashader. Depending on the size of the dataset and your machine, updating the plot might take a few seconds." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pts = hv.Points(df, vdims=['category']).redim.range(**loc['Continental US'])\n", "points = hd.dynspread(hd.datashade(pts, aggregator=ds.count_cat('category'), color_key=color_key))\n", "\n", "tiles = hv.element.tiles.ESRI().opts(alpha=0.4, bgcolor=\"black\").opts(responsive=True, min_height=600)\n", @@ -259,11 +351,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Clearly, vessel behavior is highly dependent on category, with very different patterns of motion between these categories (and presumably the other categories not shown). E.g. fishing vessels tend to hug the coasts in meandering patterns, while cargo vessels travel along straight lines further from the coast. If you zoom in to a river, you can see that passenger vessels tend to travel _across_ narrow waterways, while towing and cargo vessels travel _along_ them. Fishing vessels, as one would expect, travel out to open water and then cover a wide area around their initial destination. Zooming and panning (using the tools at the right) reveal other patterns at different locations and scales.\n", + "Using the color key, you can see that vessel behavior is highly dependent on category, with very different patterns of motion between these categories (and presumably the other categories not shown). E.g. fishing vessels tend to hug the coasts in meandering patterns, while cargo vessels travel along straight lines further from the coast. If you zoom in to a river, you can see that passenger vessels tend to travel _across_ narrow waterways, while towing and cargo vessels travel _along_ them. Zooming and panning (using the tools at the right) reveal other patterns at different locations and scales.\n", "\n", "# Selecting specific datapoints\n", "\n", - "Datashader renders data into a screen-sized array of values or pixels, which allows it to handle much larger volumes of data than can be sent to a web browser. What if you what to interact with the underlying data, e.g. to get information about clusters of datapoints or even individual datapoints? We can use HoloViews and Bokeh tools to watch for the x,y location of a tap, then query the underlying dataset for a ping in that region, and then then highlight it on top of the main plot." + "Datashader renders data into a screen-sized array of values or pixels, which allows it to handle much larger volumes of data than can be sent to a web browser. What if you what to interact with the underlying data, e.g. to get information about clusters of datapoints or even individual datapoints? For instance, here's a challenge: look up at the \"NY and NJ\" plot above, and you'll see some pink circles that turn out to be in the middle of New York State, far from the ocean. What could those be?\n", + "\n", + "To find out more about particular datapoints that we see, we can use HoloViews and Bokeh tools to watch for the x,y location of a tap, then query the underlying dataset for a ping in that region, and then then highlight it on top of the main plot." ] }, { @@ -298,9 +392,7 @@ "metadata": {}, "outputs": [], "source": [ - "xr, yr = ll2en([-126,-120.7], [47.5,49.5])\n", - "#xr, yr = ll2en([-78.2, -74.5], [40.6,42.0])\n", - "pts2 = hv.Points(df, vdims=['category']).redim.range(x=tuple(xr), y=tuple(yr))\n", + "pts2 = hv.Points(df, vdims=['category']).redim.range(**loc['Vancouver Area'])\n", "pointsp = hd.dynspread(hd.datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", "max_hits = pn.widgets.IntSlider(value=2, start=1, end=10, name=\"Max hits\", sizing_mode='stretch_width')\n", @@ -384,7 +476,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This app should run fine in a Jupyter notebook, but it can also be launched as a separate web server using `panel serve --port 5006 ship_traffic.ipynb`, allowing you to let other people explore this dataset as well." + "Here you should be able to explore this dataset fully, clicking on any interesting datapoints or clusters, including the \"Unknown\" circles in New York State mentioned above (which turn out to be false readings for a [US Coast Guard ship](https://en.wikipedia.org/wiki/USCGC_Myrtle_Hazard), clearly not able to be on land in those patterns; perhaps another example of a [previously reported GPS disruption](https://skytruth.org/2020/05/ais-ship-tracking-data-shows-false-vessel-tracks-circling-above-point-reyes-near-san-francisco/)). There are no doubt many other interesting patterns to discover here!\n", + "\n", + "The above app should run fine in a Jupyter notebook, but it can also be launched as a separate web server using `panel serve --port 5006 ship_traffic.ipynb`, allowing you to let other people explore this dataset as well. And you can adapt the code in this notebook to work with just about any other data that you can map onto the x and y axes of a plot, including categorical data if available." ] } ], From 467b3a032f13aa57fd260c0b2a23ad94ae6cac1a Mon Sep 17 00:00:00 2001 From: jlstevens Date: Mon, 25 Jan 2021 18:19:17 +0000 Subject: [PATCH 33/46] Fixed broken link --- ship_traffic/ship_traffic.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 234b99833..0feb4e99d 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -6,7 +6,7 @@ "source": [ "# Exploring AIS vessel-traffic data\n", "\n", - "This [Jupyter](https://jupyter.org) notebook demonstrates how to use the [Datashader](https://datashader.org)-based rendering in [HoloViews](https://holoviews.org) to explore and analyze US Coast Guard [Automatic Identification System (AIS)](https://en.wikipedia.org/wiki/Automatic_identification_system) vessel-location data. AIS data includes vessels identified by their [Maritime Mobile Service Identity](https://en.wikipedia.org/wiki/Maritime_Mobile_Service_Identity) numbers along with other data such as vessel type. Data is provided here for January 2020 (200 million datapoints), but additional months and years of data can be downloaded for US coastal areas from [marinecadastre.gov](marinehttps://marinecadastre.gov/ais), and with slight modifications the same code here should work for AIS data available for other regions. This notebook also illustrates a workflow for visualizing large categorical datasets in general, letting users interact with individual datapoints even though the data itself is never sent to the browser for plotting." + "This [Jupyter](https://jupyter.org) notebook demonstrates how to use the [Datashader](https://datashader.org)-based rendering in [HoloViews](https://holoviews.org) to explore and analyze US Coast Guard [Automatic Identification System (AIS)](https://en.wikipedia.org/wiki/Automatic_identification_system) vessel-location data. AIS data includes vessels identified by their [Maritime Mobile Service Identity](https://en.wikipedia.org/wiki/Maritime_Mobile_Service_Identity) numbers along with other data such as vessel type. Data is provided here for January 2020 (200 million datapoints), but additional months and years of data can be downloaded for US coastal areas from [marinecadastre.gov](https://marinecadastre.gov/ais), and with slight modifications the same code here should work for AIS data available for other regions. This notebook also illustrates a workflow for visualizing large categorical datasets in general, letting users interact with individual datapoints even though the data itself is never sent to the browser for plotting." ] }, { From 4c18e1844314aa480b1b909f8ebe7d885d028328 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Mon, 25 Jan 2021 18:19:52 +0000 Subject: [PATCH 34/46] Centered photos and improved formatting of MMSI value --- ship_traffic/ship_traffic.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 0feb4e99d..644819cd0 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -434,10 +434,10 @@ " url = get_photo_url(mmsi)\n", " response = requests.get(url, stream=True)\n", " im = Image.open(response.raw)\n", - " photos += [pn.Column(mmsi,im)] \n", + " photos += [pn.Column('MMSI: %s' % mmsi,im)] \n", " except:\n", " pass\n", - " return pn.Row(*photos)" + " return pn.Row(*([pn.Spacer(sizing_mode='stretch_width')]+photos+[pn.Spacer(sizing_mode='stretch_width')]))" ] }, { From 6c62014b95231ac26e5cf86475011709d3b6f050 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Tue, 26 Jan 2021 18:26:58 +0000 Subject: [PATCH 35/46] Updated notebook to use improved inspector API --- ship_traffic/ship_traffic.ipynb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 644819cd0..5da0ec16c 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -379,11 +379,14 @@ "ship_url = 'https://tinyurl.com/aispage/mmsi:'\n", "def full_vessel_record(df, records_to_return=2):\n", " \"Given a dataframe that includes MMSI values, return an augmented dataframe with URL and vessel info included\"\n", + " if not len(df.columns):\n", + " return None\n", " df_with_info = df.iloc[:records_to_return].merge(vessels_df, on='MMSI')\n", " df_with_types = df_with_info.merge(vessel_types, how='left', left_on='VesselType', right_on='num')[columns]\n", " df_with_types['URL'] = df_with_types.MMSI.apply(lambda x: f'{ship_url}{x}')\n", " df_with_types.VesselType = df_with_types.VesselType.apply(format_vessel_type)\n", - " return pd.DataFrame(df_with_types).drop_duplicates()" + " result = pd.DataFrame(df_with_types).drop_duplicates()\n", + " return pn.pane.DataFrame(result, index=False, render_links=True, na_rep='', sizing_mode='stretch_width')" ] }, { @@ -397,8 +400,10 @@ "\n", "max_hits = pn.widgets.IntSlider(value=2, start=1, end=10, name=\"Max hits\", sizing_mode='stretch_width')\n", "\n", - "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], \n", - " points_transformer=partial_vessel_record, hits_transformer=pn.bind(full_vessel_record, records_to_return=max_hits))\n", + "\n", + "\n", + "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap],transform=partial_vessel_record)\n", + "\n", "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", " size=10, fill_alpha=0)\n", "\n", @@ -461,15 +466,15 @@ "overlay = (tiles.apply.opts(alpha=map_opacity) *\n", " pointsp.apply.opts(alpha=data_opacity) *\n", " labels.apply.opts(alpha=label_opacity) * highlight * legend)\n", - "table = pn.bind(pn.pane.DataFrame, object=highlighter.param.hits, \n", - " index=False, render_links=True, na_rep='', sizing_mode='stretch_width')\n", + "\n", "\n", "pn.Column(\"## US AIS vessel traffic data, Jan 2020\\n\"\n", " \"Zoom or pan to explore the data, then click to select \"\n", " \"a particular data point to see more information about it (after a delay). \"\n", " \"You may need to zoom in before a point is selectable.\",\n", " pn.Row(map_opacity, data_opacity, label_opacity, max_hits),\n", - " overlay, table, photos, sizing_mode='stretch_width').servable()" + " overlay, pn.bind(full_vessel_record, df=highlighter.param.hits, records_to_return=max_hits), \n", + " photos, sizing_mode='stretch_width').servable()" ] }, { From 94093c29a36b8cb8974fc6186b3cfa1109b797d0 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Tue, 26 Jan 2021 20:09:02 +0100 Subject: [PATCH 36/46] Update ship_traffic to use latest inspect API --- ship_traffic/ship_traffic.ipynb | 64 ++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 5da0ec16c..54c204523 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -374,19 +374,7 @@ " return f'{num:.0f} ({vessel_types.loc[num].desc})'\n", "\n", "def partial_vessel_record(df):\n", - " return df.iloc[:1].merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]\n", - "\n", - "ship_url = 'https://tinyurl.com/aispage/mmsi:'\n", - "def full_vessel_record(df, records_to_return=2):\n", - " \"Given a dataframe that includes MMSI values, return an augmented dataframe with URL and vessel info included\"\n", - " if not len(df.columns):\n", - " return None\n", - " df_with_info = df.iloc[:records_to_return].merge(vessels_df, on='MMSI')\n", - " df_with_types = df_with_info.merge(vessel_types, how='left', left_on='VesselType', right_on='num')[columns]\n", - " df_with_types['URL'] = df_with_types.MMSI.apply(lambda x: f'{ship_url}{x}')\n", - " df_with_types.VesselType = df_with_types.VesselType.apply(format_vessel_type)\n", - " result = pd.DataFrame(df_with_types).drop_duplicates()\n", - " return pn.pane.DataFrame(result, index=False, render_links=True, na_rep='', sizing_mode='stretch_width')" + " return df.iloc[:1].merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]" ] }, { @@ -400,9 +388,7 @@ "\n", "max_hits = pn.widgets.IntSlider(value=2, start=1, end=10, name=\"Max hits\", sizing_mode='stretch_width')\n", "\n", - "\n", - "\n", - "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap],transform=partial_vessel_record)\n", + "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], transform=partial_vessel_record)\n", "\n", "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", " size=10, fill_alpha=0)\n", @@ -431,18 +417,30 @@ " ship_id =ship_id[0].replace('shipid:','')\n", " return f\"https://photos.marinetraffic.com/ais/showphoto.aspx?shipid={ship_id}&size=thumb300&stamp=false\"\n", "\n", - "def get_photos(df=None):\n", + "def get_photos(df=None, n_records=2):\n", " photos = []\n", " if df is not None and 'MMSI' in df.columns:\n", - " for mmsi in df.MMSI.to_list():\n", + " for mmsi in df.iloc[:n_records].MMSI.to_list():\n", " try:\n", " url = get_photo_url(mmsi)\n", " response = requests.get(url, stream=True)\n", " im = Image.open(response.raw)\n", " photos += [pn.Column('MMSI: %s' % mmsi,im)] \n", - " except:\n", + " except Exception:\n", " pass\n", - " return pn.Row(*([pn.Spacer(sizing_mode='stretch_width')]+photos+[pn.Spacer(sizing_mode='stretch_width')]))" + " return pn.Row(*([pn.Spacer(sizing_mode='stretch_width')]+photos+[pn.Spacer(sizing_mode='stretch_width')]))\n", + "\n", + "ship_url = 'https://tinyurl.com/aispage/mmsi:'\n", + "def full_vessel_record(df, n_records=2):\n", + " \"Given a dataframe that includes MMSI values, return an augmented dataframe with URL and vessel info included\"\n", + " if not len(df.columns):\n", + " return None\n", + " df_with_info = df.iloc[:n_records].merge(vessels_df, on='MMSI')\n", + " df_with_types = df_with_info.merge(vessel_types, how='left', left_on='VesselType', right_on='num')[columns]\n", + " df_with_types['URL'] = df_with_types.MMSI.apply(lambda x: f'{ship_url}{x}')\n", + " df_with_types.VesselType = df_with_types.VesselType.apply(format_vessel_type)\n", + " result = pd.DataFrame(df_with_types).drop_duplicates()\n", + " return pn.pane.DataFrame(result, index=False, render_links=True, na_rep='', sizing_mode='stretch_width')" ] }, { @@ -458,7 +456,8 @@ "metadata": {}, "outputs": [], "source": [ - "photos = pn.bind(get_photos, df=highlighter.param.hits)\n", + "photos = pn.bind(get_photos, df=highlighter.param.hits, n_records=max_hits)\n", + "table = pn.bind(full_vessel_record, df=highlighter.param.hits, n_records=max_hits)\n", "sopts = dict(start=0, end=1, sizing_mode='stretch_width')\n", "map_opacity = pn.widgets.FloatSlider(value=0.7, name=\"Map opacity\", **sopts)\n", "data_opacity = pn.widgets.FloatSlider(value=1.0, name=\"Data opacity\", **sopts)\n", @@ -467,14 +466,21 @@ " pointsp.apply.opts(alpha=data_opacity) *\n", " labels.apply.opts(alpha=label_opacity) * highlight * legend)\n", "\n", - "\n", - "pn.Column(\"## US AIS vessel traffic data, Jan 2020\\n\"\n", - " \"Zoom or pan to explore the data, then click to select \"\n", - " \"a particular data point to see more information about it (after a delay). \"\n", - " \"You may need to zoom in before a point is selectable.\",\n", - " pn.Row(map_opacity, data_opacity, label_opacity, max_hits),\n", - " overlay, pn.bind(full_vessel_record, df=highlighter.param.hits, records_to_return=max_hits), \n", - " photos, sizing_mode='stretch_width').servable()" + "description = \"\"\"\n", + "## US AIS vessel traffic data, Jan 2020\n", + "\n", + "Zoom or pan to explore the data, then click to select\n", + "a particular data point to see more information about\n", + "it (after a delay). You may need to zoom in before a\n", + "point is selectable.\n", + "\"\"\"\n", + "\n", + "pn.Column(\n", + " description,\n", + " pn.Row(map_opacity, data_opacity, label_opacity, max_hits),\n", + " overlay, table, photos,\n", + " sizing_mode='stretch_width'\n", + ").servable()" ] }, { From 4eda90e0945eb4f051faafcf9d054ab28bc4be34 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Tue, 26 Jan 2021 23:40:23 -0600 Subject: [PATCH 37/46] Fix duplicate photos for repeated points from same ship --- ship_traffic/ship_traffic.ipynb | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 54c204523..6ed18ffef 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -355,7 +355,7 @@ "\n", "# Selecting specific datapoints\n", "\n", - "Datashader renders data into a screen-sized array of values or pixels, which allows it to handle much larger volumes of data than can be sent to a web browser. What if you what to interact with the underlying data, e.g. to get information about clusters of datapoints or even individual datapoints? For instance, here's a challenge: look up at the \"NY and NJ\" plot above, and you'll see some pink circles that turn out to be in the middle of New York State, far from the ocean. What could those be?\n", + "Datashader renders data into a screen-sized array of values or pixels, which allows it to handle much larger volumes of data than can be sent to a web browser. What if you what to interact with the underlying data, e.g. to get information about clusters of datapoints or even individual datapoints? For instance, here's a challenge: look up at the \"NY and NJ\" plot above, and you'll see some pink circles and lines that turn out to be in the middle of New York State and Pennsylvania, far from the ocean. What could those be?\n", "\n", "To find out more about particular datapoints that we see, we can use HoloViews and Bokeh tools to watch for the x,y location of a tap, then query the underlying dataset for a ping in that region, and then then highlight it on top of the main plot." ] @@ -373,7 +373,7 @@ " if np.isnan(num): num = 0\n", " return f'{num:.0f} ({vessel_types.loc[num].desc})'\n", "\n", - "def partial_vessel_record(df):\n", + "def brief_vessel_record(df):\n", " return df.iloc[:1].merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]" ] }, @@ -388,7 +388,7 @@ "\n", "max_hits = pn.widgets.IntSlider(value=2, start=1, end=10, name=\"Max hits\", sizing_mode='stretch_width')\n", "\n", - "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], transform=partial_vessel_record)\n", + "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], transform=brief_vessel_record)\n", "\n", "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", " size=10, fill_alpha=0)\n", @@ -400,7 +400,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We could view the result above by uncommenting the last line, but let's just go ahead and make a little [Panel](https://panel.holoviz.org) app so we can add a few extra interactive features. First, some code to fetch a photo of the selected vessels, if available:" + "We could view the result above by uncommenting the last line, but let's just go ahead and make a little [Panel](https://panel.holoviz.org) app so we can add a few extra interactive features. First, some code to fetch a photo of the selected vessels, if available, plus additional info about each vessel:" ] }, { @@ -420,7 +420,7 @@ "def get_photos(df=None, n_records=2):\n", " photos = []\n", " if df is not None and 'MMSI' in df.columns:\n", - " for mmsi in df.iloc[:n_records].MMSI.to_list():\n", + " for mmsi in df.iloc[:n_records].MMSI.drop_duplicates().to_list():\n", " try:\n", " url = get_photo_url(mmsi)\n", " response = requests.get(url, stream=True)\n", @@ -475,19 +475,16 @@ "point is selectable.\n", "\"\"\"\n", "\n", - "pn.Column(\n", - " description,\n", + "pn.Column(description,\n", " pn.Row(map_opacity, data_opacity, label_opacity, max_hits),\n", - " overlay, table, photos,\n", - " sizing_mode='stretch_width'\n", - ").servable()" + " overlay, table, photos, sizing_mode='stretch_width').servable()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Here you should be able to explore this dataset fully, clicking on any interesting datapoints or clusters, including the \"Unknown\" circles in New York State mentioned above (which turn out to be false readings for a [US Coast Guard ship](https://en.wikipedia.org/wiki/USCGC_Myrtle_Hazard), clearly not able to be on land in those patterns; perhaps another example of a [previously reported GPS disruption](https://skytruth.org/2020/05/ais-ship-tracking-data-shows-false-vessel-tracks-circling-above-point-reyes-near-san-francisco/)). There are no doubt many other interesting patterns to discover here!\n", + "Here you should be able to explore this dataset fully, clicking on any interesting datapoints or clusters, including the \"Unknown\" circles in New York State and Pennsylvania mentioned above (which turn out to be false readings for a [US Coast Guard ship based in Guam](https://en.wikipedia.org/wiki/USCGC_Myrtle_Hazard), clearly not able to be on land in those patterns; perhaps another example of a [previously reported GPS disruption](https://skytruth.org/2020/05/ais-ship-tracking-data-shows-false-vessel-tracks-circling-above-point-reyes-near-san-francisco/)). There are no doubt many other interesting patterns to discover here!\n", "\n", "The above app should run fine in a Jupyter notebook, but it can also be launched as a separate web server using `panel serve --port 5006 ship_traffic.ipynb`, allowing you to let other people explore this dataset as well. And you can adapt the code in this notebook to work with just about any other data that you can map onto the x and y axes of a plot, including categorical data if available." ] From fe45be710de3527f6161dd8bf4e933c2df7760b9 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 27 Jan 2021 11:14:53 -0600 Subject: [PATCH 38/46] Set up test data and initial tap location --- ship_traffic/ship_traffic.ipynb | 29 +++++++++++------------ test_data/ship_traffic/AIS_2020_01_01.csv | 10 ++++++++ test_data/ship_traffic/AIS_2020_01_02.csv | 10 ++++++++ 3 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 test_data/ship_traffic/AIS_2020_01_01.csv create mode 100644 test_data/ship_traffic/AIS_2020_01_02.csv diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 6ed18ffef..a88ea70a4 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -151,10 +151,10 @@ "metadata": {}, "outputs": [], "source": [ - "basedir = './2020/'\n", + "basedir = './data/ship_traffic/'\n", "basename = 'AIS_2020_01'\n", "index = 'MMSI'\n", - "dfcols = ['MMSI', 'LON', 'LAT', 'BaseDateTime', 'VesselType']\n", + "dfcols = ['MMSI', 'LON', 'LAT', 'VesselType']\n", "vesselcols = ['MMSI', 'IMO', 'CallSign', 'VesselName', 'VesselType', 'Length', 'Width']\n", "\n", "def load_data(spatial_index=False):\n", @@ -163,7 +163,7 @@ " \n", " if (os.path.exists(cache_file) and os.path.exists(vessels_file)):\n", " print('Reading vessel info file')\n", - " vessels = dd.read_parquet(vessels_file)\n", + " vessels = dd.read_parquet(vessels_file).compute()\n", "\n", " print('Reading parquet file')\n", " gdf = sp.io.read_parquet_dask(cache_file).persist()\n", @@ -223,7 +223,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.head()" + "df.head(1)" ] }, { @@ -366,7 +366,6 @@ "metadata": {}, "outputs": [], "source": [ - "vessels_df = vessels.compute()\n", "columns = ['VesselName', 'MMSI', 'IMO', 'VesselType']\n", "\n", "def format_vessel_type(num):\n", @@ -374,7 +373,7 @@ " return f'{num:.0f} ({vessel_types.loc[num].desc})'\n", "\n", "def brief_vessel_record(df):\n", - " return df.iloc[:1].merge(vessels_df, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]" + " return df.iloc[:1].merge(vessels, on='MMSI').merge(vessel_types, on='category')[['geometry']+columns]" ] }, { @@ -383,15 +382,15 @@ "metadata": {}, "outputs": [], "source": [ - "pts2 = hv.Points(df, vdims=['category']).redim.range(**loc['Vancouver Area'])\n", - "pointsp = hd.dynspread(hd.datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", + "pts2 = hv.Points(df, vdims=['category']).redim.range(**loc['Vancouver Area'])\n", + "pointsp = hd.dynspread(hd.datashade(pts2, color_key=color_key, aggregator=ds.count_cat('category'), min_alpha=90))\n", "\n", - "max_hits = pn.widgets.IntSlider(value=2, start=1, end=10, name=\"Max hits\", sizing_mode='stretch_width')\n", + "max_hits = pn.widgets.IntSlider(value=2, start=1, end=10, name=\"Max hits\", sizing_mode='stretch_width')\n", + "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], transform=brief_vessel_record,\n", + " x=-13922122, y=6184391) # optional initial values for static web page\n", "\n", - "highlighter = hd.inspect_points.instance(streams=[hv.streams.Tap], transform=brief_vessel_record)\n", - "\n", - "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", - " size=10, fill_alpha=0)\n", + "highlight = highlighter(pointsp).opts(color='white', tools=[\"hover\"], marker='square', \n", + " size=10, fill_alpha=0)\n", "\n", "#tiles * pointsp * highlight * legend" ] @@ -432,10 +431,10 @@ "\n", "ship_url = 'https://tinyurl.com/aispage/mmsi:'\n", "def full_vessel_record(df, n_records=2):\n", - " \"Given a dataframe that includes MMSI values, return an augmented dataframe with URL and vessel info included\"\n", + " \"Given a dataframe that includes MMSI values, return with URL, vessel info added\"\n", " if not len(df.columns):\n", " return None\n", - " df_with_info = df.iloc[:n_records].merge(vessels_df, on='MMSI')\n", + " df_with_info = df.iloc[:n_records].merge(vessels, on='MMSI')\n", " df_with_types = df_with_info.merge(vessel_types, how='left', left_on='VesselType', right_on='num')[columns]\n", " df_with_types['URL'] = df_with_types.MMSI.apply(lambda x: f'{ship_url}{x}')\n", " df_with_types.VesselType = df_with_types.VesselType.apply(format_vessel_type)\n", diff --git a/test_data/ship_traffic/AIS_2020_01_01.csv b/test_data/ship_traffic/AIS_2020_01_01.csv new file mode 100644 index 000000000..3a65ed2be --- /dev/null +++ b/test_data/ship_traffic/AIS_2020_01_01.csv @@ -0,0 +1,10 @@ +MMSI,BaseDateTime,LAT,LON,SOG,COG,Heading,VesselName,IMO,CallSign,VesselType,Status,Length,Width,Draft,Cargo,TranscieverClass +367149340,2020-01-01T00:00:00,29.96476,-90.02724,1.3,10.0,16.0,SYDNEE TAYLOR,,WDD4807,31,0,26,9,,31,B +367687520,2020-01-01T00:00:00,30.20558,-91.03578,10.7,124.9,130.0,CHIPPEWA,,WDI3361,31,0,22,,,,B +367368170,2020-01-01T00:00:03,47.53785,-122.32833,0.6,46.5,141.0,SONJA H,,WDE5536,31,0,18,6,,32,B +367007980,2020-01-01T00:00:05,37.95154,-121.32682,0.0,-49.6,511.0,ANGIE M BRUSCO,IMO5111359,WDC3446,31,0,28,7,3.4,,B +367538940,2020-01-01T00:00:05,30.00258,-93.22608,3.1,168.0,511.0,RITA ANN,,WDG4670,31,0,21,,,,B +367054790,2020-01-01T00:00:07,29.76044,-95.10476,0.1,-180.8,511.0,COLT,,WDC6330,31,15,20,7,,31,B +12345678,2020-01-01T00:00:06,30.05201,-90.54060,0.0,162.2,511.0,INGRAM TEST UNIT,,WWW0000,32,0,46,14,3.5,32,B +367567020,2020-01-01T00:00:08,34.14870,-119.20176,0.0,0.0,511.0,LULAPIN,IMO8997869,WDG7412,31,0,23,8,,,B +367389480,2020-01-01T00:00:11,30.69094,-81.45967,0.0,-157.8,511.0,MAVERICK,,WDE7139,31,0,18,6,,52,B diff --git a/test_data/ship_traffic/AIS_2020_01_02.csv b/test_data/ship_traffic/AIS_2020_01_02.csv new file mode 100644 index 000000000..db3c31c94 --- /dev/null +++ b/test_data/ship_traffic/AIS_2020_01_02.csv @@ -0,0 +1,10 @@ +MMSI,BaseDateTime,LAT,LON,SOG,COG,Heading,VesselName,IMO,CallSign,VesselType,Status,Length,Width,Draft,Cargo,TranscieverClass +368119660,2020-01-02T00:00:00,40.66729,-74.00783,0.0,-49.6,17.0,,,,,0,,,,,B +368009250,2020-01-02T00:00:00,47.66543,-122.39625,0.1,-156.9,109.0,,,,,0,,,,,A +368114180,2020-01-02T00:00:00,29.83139,-90.06786,0.0,-49.6,511.0,,,,,0,,,,,B +368010960,2020-01-02T00:00:00,37.04289,-89.17536,5.8,123.4,123.0,,,,,0,,,,,A +368090830,2020-01-02T00:00:00,27.84044,-97.08588,0.0,0.0,511.0,,,,,0,,,,,B +636018561,2020-01-02T00:00:01,34.13566,-76.47055,6.0,-178.9,236.0,,,,,0,,,,,B +563056600,2020-01-02T00:00:01,48.47790,-124.90163,9.9,86.6,89.0,,,,,0,,,,,A +367790680,2020-01-02T00:00:02,55.05991,-162.32734,0.0,197.0,511.0,,,,,15,,,,,B +368029640,2020-01-02T00:00:02,40.71516,-74.01818,0.0,-49.6,511.0,,,,,0,,,,,B From 6cb074ae30d7f090137344a532a3016c5c8ff368 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 27 Jan 2021 11:19:20 -0600 Subject: [PATCH 39/46] Show legend before categorical plots --- ship_traffic/ship_traffic.ipynb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index a88ea70a4..295834bf9 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -106,8 +106,7 @@ " enumerate(colors[:(len(groups))][::-1])}\n", "legend = hv.NdOverlay({groups[k]: hv.Points([0,0], label=str(groups[k])).opts(\n", " color=cc.rgb_to_hex(*v), size=0) \n", - " for k, v in color_key.items()})\n", - "#legend # uncomment to see legend alone" + " for k, v in color_key.items()})" ] }, { @@ -296,7 +295,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Even more structure is visible if we color by the vessel category (using the color key shown in later Bokeh plots):" + "Even more structure is visible if we color by the vessel category using the color key we defined:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "legend" ] }, { From 7d564656f9e9c6d71e5437d4cceadd81816dc8b0 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 27 Jan 2021 11:26:12 -0600 Subject: [PATCH 40/46] Set up data access --- ship_traffic/anaconda-project.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ship_traffic/anaconda-project.yml b/ship_traffic/anaconda-project.yml index e652e600d..1198219dd 100644 --- a/ship_traffic/anaconda-project.yml +++ b/ship_traffic/anaconda-project.yml @@ -48,9 +48,12 @@ commands: variables: {} downloads: - DATAFILE: - url: https://zenodo.org/record/3541812/files/HG_OOSTENDE-gps-2018.csv - filename: data/HG_OOSTENDE-gps-2018.csv + DATA: + url: http://s3.amazonaws.com/datashader-data/ship_traffic.zip + description: | + US AIS records from 1/2020 + filename: data/AIS_2020_01_broadcast.parq + unzip: true env_specs: default: {} From 0c176d865afdbe8d486beb7bb008e0b55c108ae0 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 27 Jan 2021 11:31:21 -0600 Subject: [PATCH 41/46] Fixed path --- ship_traffic/ship_traffic.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 295834bf9..4bb6f3770 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -150,7 +150,7 @@ "metadata": {}, "outputs": [], "source": [ - "basedir = './data/ship_traffic/'\n", + "basedir = './data/'\n", "basename = 'AIS_2020_01'\n", "index = 'MMSI'\n", "dfcols = ['MMSI', 'LON', 'LAT', 'VesselType']\n", From 3a57799e6e7af587d4f52ab0b7e8838a49fcd40a Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 27 Jan 2021 12:37:02 -0600 Subject: [PATCH 42/46] Pinned to dev releases --- ship_traffic/anaconda-project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ship_traffic/anaconda-project.yml b/ship_traffic/anaconda-project.yml index 1198219dd..5a436fcec 100644 --- a/ship_traffic/anaconda-project.yml +++ b/ship_traffic/anaconda-project.yml @@ -17,14 +17,14 @@ packages: &pkgs - colorcet ==2 - dask ==2020.12.0 - datashader ==0.12.0 -- holoviews ==1.14.0 +- holoviews ==1.14.2a1 - notebook ==6.1.5 - numba ==0.51.2 - numexpr ==2.7.1 - pandas ==1.1.5 - panel ==0.10.3 - python ==3.7.9 -- spatialpandas ==0.3.6 +- spatialpandas ==v0.4.0a1 - xarray ==0.16.2 - pip ==20.3.3 - conda-forge::pyarrow ==2 From 3578877c36a4fc5f478d4b585be7e4c9a07d1525 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 27 Jan 2021 12:40:16 -0600 Subject: [PATCH 43/46] Use pyviz/label/dev --- ship_traffic/anaconda-project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ship_traffic/anaconda-project.yml b/ship_traffic/anaconda-project.yml index 5a436fcec..c77a6b0fd 100644 --- a/ship_traffic/anaconda-project.yml +++ b/ship_traffic/anaconda-project.yml @@ -10,7 +10,7 @@ labels: user_fields: [labels, skip, maintainers] channels: -- pyviz +- pyviz/label/dev packages: &pkgs - bokeh ==2.2.3 From c267893d28fda072cb22ab30dd254879b031e3cb Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Wed, 27 Jan 2021 12:44:45 -0600 Subject: [PATCH 44/46] Fix typo in .yml --- ship_traffic/anaconda-project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ship_traffic/anaconda-project.yml b/ship_traffic/anaconda-project.yml index c77a6b0fd..13169b554 100644 --- a/ship_traffic/anaconda-project.yml +++ b/ship_traffic/anaconda-project.yml @@ -7,7 +7,7 @@ labels: - datashader - holoviews -user_fields: [labels, skip, maintainers] +user_fields: [labels, skip, maintainers, user_fields] channels: - pyviz/label/dev @@ -24,7 +24,7 @@ packages: &pkgs - pandas ==1.1.5 - panel ==0.10.3 - python ==3.7.9 -- spatialpandas ==v0.4.0a1 +- spatialpandas ==0.4.0a1 - xarray ==0.16.2 - pip ==20.3.3 - conda-forge::pyarrow ==2 From 495dbda22bfc859f6a253017ab97decc2b1760fa Mon Sep 17 00:00:00 2001 From: jlstevens Date: Wed, 27 Jan 2021 19:05:14 +0000 Subject: [PATCH 45/46] Fixed flakes --- ship_traffic/ship_traffic.ipynb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ship_traffic/ship_traffic.ipynb b/ship_traffic/ship_traffic.ipynb index 4bb6f3770..7a1db4d7e 100644 --- a/ship_traffic/ship_traffic.ipynb +++ b/ship_traffic/ship_traffic.ipynb @@ -16,11 +16,10 @@ "outputs": [], "source": [ "import os, requests, numpy as np, pandas as pd, holoviews as hv, holoviews.operation.datashader as hd\n", - "import dask.dataframe as dd, panel as pn, panel.widgets as pnw, colorcet as cc, datashader as ds\n", + "import dask.dataframe as dd, panel as pn, colorcet as cc, datashader as ds\n", "import spatialpandas as sp, spatialpandas.io, spatialpandas.geometry, spatialpandas.dask\n", "\n", "from PIL import Image\n", - "from glob import glob\n", "from holoviews.util.transform import lon_lat_to_easting_northing as ll2en\n", "from dask.diagnostics import ProgressBar\n", "\n", From 118dc507e04758d4ec7f2998b7dde7e2c9356beb Mon Sep 17 00:00:00 2001 From: jlstevens Date: Wed, 27 Jan 2021 19:26:45 +0000 Subject: [PATCH 46/46] Temporarily skipping small_data_cleanup step --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 322dc9272..e863dd5a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: if doit changes_in_dir --name $DIR; then doit small_data_setup --name $DIR doit test_project --name $DIR - doit small_data_cleanup --name $DIR + # doit small_data_cleanup --name $DIR fi; fi; done