Merge pull request #109 from astrolabsoftware/0.3.0

Bump version: 0.2.2 --> 0.3.0
astrolabsoftware · Nov 23, 2018 · 19873df · 19873df
2 parents 8b35c67 + 5e19ffd
commit 19873df
Show file tree

Hide file tree

Showing 65 changed files with 70 additions and 534 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,15 @@
+## 0.3.0
+
+- Complete change of the interface ([PR](https://github.com/astrolabsoftware/spark3D/pull/108)). This includes:
+  - DataFrame API
+  - Extension of the Spark SQL module (new methods + implicits on DataFrame)
+  - Metadata are propagated (not just 3D coordinates).
+  - Seamless interface with Python
+- Compiling against Apache Spark 2.3.2: fix incompatibilities ([PR](https://github.com/astrolabsoftware/spark3D/pull/107))
+- Website update: spark-fits version + vulnerability fix ([PR](https://github.com/astrolabsoftware/spark3D/pull/105))
+-  Fix wrong angle definition in cartesian to spherical coordinate change! ([PR](https://github.com/astrolabsoftware/spark3D/pull/102))
+
+
 ## 0.2.2
 
 - Add the conversion from FITS file (single HDU) to parquet ([PR](https://github.com/astrolabsoftware/spark3D/pull/92))

diff --git a/README.md b/README.md
@@ -12,6 +12,7 @@
 - [07/2018] **Release**: version 0.1.3, 0.1.4, 0.1.5
 - [08/2018] **Release**: version 0.2.0, 0.2.1 (pyspark3d)
 - [09/2018] **Release**: version 0.2.2
+- [11/2018] **Release**: version 0.3.0 (new DataFrame API)
 
 <p align="center"><img width="300" src="https://github.com/astrolabsoftware/spark3D/raw/master/pic/spark3d_newapi.png"/>
 </p>

diff --git a/build.sbt b/build.sbt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2018 Julien Peloton
+ * Copyright 2018 AstroLab Software
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,7 +19,7 @@ import xerial.sbt.Sonatype._
 lazy val root = (project in file(".")).
  settings(
    inThisBuild(List(
-     version      := "0.2.2"
+     version      := "0.3.0"
    )),
    // Name of the application
    name := "spark3D",
@@ -46,7 +46,7 @@ lazy val root = (project in file(".")).
      "org.apache.spark" %% "spark-core" % "2.3.2" % "provided",
      "org.apache.spark" %% "spark-sql" % "2.3.2" % "provided",
      // For loading FITS files
-     "com.github.astrolabsoftware" %% "spark-fits" % "0.7.0",
+     "com.github.astrolabsoftware" %% "spark-fits" % "0.7.1",
      // "org.datasyslab" % "geospark" % "1.1.3",
      // Uncomment if you want to trigger visualisation
      // "com.github.haifengl" % "smile-plot" % "1.5.1",

diff --git a/docs/01_installation.md b/docs/01_installation.md
@@ -20,7 +20,7 @@ another version, feel free to contact us. In addition to Spark, the library has
 You can link spark3D to your project (either `spark-shell` or `spark-submit`) by specifying the coordinates:
 
 ```bash
-toto:~$ spark-submit --packages "com.github.astrolabsoftware:spark3d_2.11:0.2.2" <...>
+toto:~$ spark-submit --packages "com.github.astrolabsoftware:spark3d_2.11:0.3.0" <...>
 ```
 
 It might not contain the latest features though (see *Building from source*).
@@ -69,7 +69,7 @@ result on the screen, plus details of the coverage at
 First produce a jar of the spark3D library, and then launch a spark-shell by specifying the external dependencies:
 
 ```bash
-toto:~$ JARS="target/scala-2.11/spark3d_2.11-0.2.2.jar,lib/jhealpix.jar"
+toto:~$ JARS="target/scala-2.11/spark3d_2.11-0.3.0.jar,lib/jhealpix.jar"
 toto:~$ PACKAGES="com.github.astrolabsoftware:spark-fits_2.11:0.7.0"
 toto:~$ spark-shell --jars $JARS --packages $PACKAGES
 ```
@@ -83,7 +83,7 @@ scala> // etc...
 Note that if you make a fat jar (that is building with `sbt ++${SCALA_VERSION} assembly` and not `sbt ++${SCALA_VERSION} package`), you do not need to specify external dependencies as they are already included in the resulting jar:
 
 ```bash
-toto:~$ FATJARS="target/scala-2.11/spark3D-assembly-0.2.2.jar"
+toto:~$ FATJARS="target/scala-2.11/spark3D-assembly-0.3.0.jar"
 toto:~$ spark-shell --jars $FATJARS
 ```
 
@@ -188,7 +188,7 @@ and then launch a pyspark shell:
 
 ```bash
 toto:~$ PYSPARK_DRIVER_PYTHON=ipython pyspark \
-  --jars /path/to/target/scala-2.11/spark3D-assembly-0.2.2.jar
+  --jars /path/to/target/scala-2.11/spark3D-assembly-0.3.0.jar
 ```
 
 # Batch mode and provided examples

diff --git a/docs/_pages/home.md b/docs/_pages/home.md
@@ -8,7 +8,7 @@ header:
   cta_url: "/docs/installation/"
   caption:
 intro:
-  - excerpt: '<p><font size="6">Spark extension for processing large-scale 3D data sets: Astrophysics, High Energy Physics, Meteorology, ...</font></p><br /><a href="https://github.com/astrolabsoftware/spark3D/releases/tag/0.2.2">Latest release v0.2.2</a>'
+  - excerpt: '<p><font size="6">Spark extension for processing large-scale 3D data sets: Astrophysics, High Energy Physics, Meteorology, ...</font></p><br /><a href="https://github.com/astrolabsoftware/spark3D/releases/tag/0.3.0">Latest release v0.3.0</a>'
 excerpt: '{::nomarkdown}<iframe style="display: inline-block;" src="https://ghbtns.com/github-btn.html?user=astrolabsoftware&repo=spark3D&type=star&count=true&size=large" frameborder="0" scrolling="0" width="160px" height="30px"></iframe> <iframe style="display: inline-block;" src="https://ghbtns.com/github-btn.html?user=astrolabsoftware&repo=spark3D&type=fork&count=true&size=large" frameborder="0" scrolling="0" width="158px" height="30px"></iframe>{:/nomarkdown}'
 feature_row:
   - image_path:

diff --git a/pyspark3d/__init__.py b/pyspark3d/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Julien Peloton
+# Copyright 2018 AstroLab Software
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/pyspark3d/checkers.py b/pyspark3d/checkers.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Julien Peloton
+# Copyright 2018 AstroLab Software
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/pyspark3d/converters.py b/pyspark3d/converters.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Julien Peloton
+# Copyright 2018 AstroLab Software
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/pyspark3d/geometryObjects.py b/pyspark3d/geometryObjects.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Julien Peloton
+# Copyright 2018 AstroLab Software
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/pyspark3d/pyspark3d_conf.py b/pyspark3d/pyspark3d_conf.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Julien Peloton
+# Copyright 2018 AstroLab Software
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/pyspark3d/repartitioning.py b/pyspark3d/repartitioning.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Julien Peloton
+# Copyright 2018 AstroLab Software
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/pyspark3d/version.py b/pyspark3d/version.py
@@ -1,4 +1,4 @@
 # pyspark3d and spark3D have the same version number
-__version__ = "0.2.2"
+__version__ = "0.3.0"
 __scala_version__ = "2.11"
 __scala_version_all__ = "2.11.8"
diff --git a/pyspark3d/visualisation.py b/pyspark3d/visualisation.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Julien Peloton
+# Copyright 2018 AstroLab Software
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/runners/README.md b/runners/README.md
diff --git a/runners/benchmark_knn.sh b/runners/benchmark_knn.sh
diff --git a/runners/launch_pyspark.sh b/runners/launch_pyspark.sh
diff --git a/runners/run_knn.sh b/runners/run_knn.sh
diff --git a/runners/run_knnGeo.sh b/runners/run_knnGeo.sh
diff --git a/runners/run_part.sh b/runners/run_part.sh