Merge pull request #66 from astrolabsoftware/packageName

Package name

CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.1.3
+
+- Add KNN routines ([KNN](https://github.com/astrolabsoftware/spark3D/pull/59), [KNN](https://github.com/astrolabsoftware/spark3D/pull/60), [KNN](https://github.com/astrolabsoftware/spark3D/pull/62))
+- Unify API to load data ([Point3DRDD](https://github.com/astrolabsoftware/spark3D/pull/63), [SphereRDD](https://github.com/astrolabsoftware/spark3D/pull/64))
+- Speed-up cross-match methods by using native Scala methods ([Scala](https://github.com/astrolabsoftware/spark3D/pull/58))
+- Add a new website + spark3D belongs to AstroLab Software ([website](https://astrolabsoftware.github.io/))
+- Update tutorials ([tuto](https://astrolabsoftware.github.io/spark3D/).
+- Few fixes here and there...
+
 ## 0.1.1
 
 - Add scripts to generate test data ([PR](https://github.com/astrolabsoftware/spark3D/pull/34))

README.md

@@ -9,6 +9,7 @@
 - [05/2018] **GSoC 2018**: spark3D has been selected to the Google Summer of Code (GSoC) 2018. Congratulation to [@mayurdb](https://github.com/mayurdb) who will work on the project this year!
 - [06/2018] **Release**: version 0.1.0, 0.1.1
 - [07/2018] **New location**: spark3D is an official project of [AstroLab Software](https://astrolabsoftware.github.io/)!
+- [07/2018] **Release**: version 0.1.3
 
 ## Installation and tutorials
 
@@ -21,3 +22,7 @@ See our amazing [website](https://astrolabsoftware.github.io/spark3D/)!
 * Mayur Bhosale (mayurdb31 at gmail.com) -- GSoC 2018.
 
 Contributing to spark3D: see [CONTRIBUTING](https://github.com/astrolabsoftware/spark3D/blob/master/CONTRIBUTING.md).
+
+## Support
+
+<p align="center"><img width="100" src="https://github.com/astrolabsoftware/spark-fits/raw/master/pic/lal_logo.jpg"/> <img width="100" src="https://github.com/astrolabsoftware/spark-fits/raw/master/pic/psud.png"/> <img width="100" src="https://github.com/astrolabsoftware/spark-fits/raw/master/pic/1012px-Centre_national_de_la_recherche_scientifique.svg.png"/></p>

build.sbt

@@ -19,8 +19,7 @@ import xerial.sbt.Sonatype._
 lazy val root = (project in file(".")).
  settings(
    inThisBuild(List(
-     version      := "0.1.2"
-     // mainClass in Compile := Some("com.sparkfits.examples.OnionSpace")
+     version      := "0.1.3"
    )),
    // Name of the application
    name := "spark3D",
@@ -36,7 +35,7 @@ lazy val root = (project in file(".")).
    // Do not publish artifact in test
    publishArtifact in Test := false,
    // Exclude runner class for the coverage
-   coverageExcludedPackages := "<empty>;com.spark3d.examples*",
+   coverageExcludedPackages := "<empty>;com.astrolabsoftware.spark3d.examples*",
    // Excluding Scala library JARs that are included in the binary Scala distribution
    // assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
    // Shading to avoid conflicts with pre-installed nom.tam.fits library
@@ -47,7 +46,7 @@ lazy val root = (project in file(".")).
      "org.apache.spark" %% "spark-core" % "2.1.0" % "provided",
      "org.apache.spark" %% "spark-sql" % "2.1.0" % "provided",
      // For loading FITS files
-     "com.github.JulienPeloton" %% "spark-fits" % "0.4.0",
+     "com.github.astrolabsoftware" %% "spark-fits" % "0.4.0",
      // "org.datasyslab" % "geospark" % "1.1.3",
      // Uncomment if you want to trigger visualisation
      // "com.github.haifengl" % "smile-plot" % "1.5.1",

docs/01_installation.md

@@ -77,7 +77,7 @@ toto:~$ spark-shell --jars $JARS --packages $PACKAGES
 You will be able to import anything from spark3D
 
 ```scala
-scala> import com.spark3d.geometryObjects.Point3D
+scala> import com.astrolabsoftware.spark3d.geometryObjects.Point3D
 scala> // etc...
 ```
 Note that if you make a fat jar (that is building with `sbt assembly` and not `sbt package`), you do not need to specify external dependencies as they are already included in the resulting jar:

docs/02_introduction.md

@@ -14,7 +14,7 @@ spark3D supports various 3D shapes: points (`Point3D`), spherical shells (`Shell
 ### Point3D
 
 ```scala
-import com.spark3d.geometryObjects.Point3D
+import com.astrolabsoftware.spark3d.geometryObjects.Point3D
 
 // Cartesian coordinates
 val points = new Point3D(x: Double, y: Double, z: Double, isSpherical: Boolean = false)
@@ -26,7 +26,7 @@ val points = new Point3D(r: Double, theta: Double, phi: Double, isSpherical: Boo
 ### Shells and Spheres
 
 ```scala
-import com.spark3d.geometryObjects.ShellEnvelope
+import com.astrolabsoftware.spark3d.geometryObjects.ShellEnvelope
 
 // Shell from 3D coordinates + inner/outer radii
 val shells = new ShellEnvelope(x: Double, y: Double, z: Double, isSpherical: Boolean, innerRadius: Double, outerRadius: Double)
@@ -44,7 +44,7 @@ val spheres = new ShellEnvelope(center: Point3D, isSpherical: Boolean, radius: D
 ### Boxes
 
 ```scala
-import com.spark3d.geometryObjects.BoxEnvelope
+import com.astrolabsoftware.spark3d.geometryObjects.BoxEnvelope
 
 // Box from region defined by three (cartesian) coordinates.
 val boxes = new BoxEnvelope(p1: Point3D, p2: Point3D, p3: Point3D)
@@ -68,7 +68,7 @@ In this tutorial we will review the steps to simply create RDD from 3D data sets
 A point is an object with 3 spatial coordinates. In spark3D, you can choose the coordinate system between cartesian `(x, y, z)` and spherical `(r, theta, phi)`. Let's suppose we have a text file (CSV, JSON, or TXT) whose columns are labeled `x`, `y` and `z`, the cartesian coordinates of points:
 
 ```scala
-import com.spark3d.spatial3DRDD.Point3DRDD
+import com.astrolabsoftware.spark3d.spatial3DRDD.Point3DRDD
 
 // We assume filename contains at least 3 columns whose names are `colnames`
 // Order of columns in the file does not matter, as they will be re-aranged
@@ -79,7 +79,7 @@ val pointRDD = new Point3DRDD(spark: SparkSession, filename: String, colnames: S
 With FITS data, with data in the HDU #1, you would just do
 
 ```scala
-import com.spark3d.spatial3DRDD.Point3DRDD
+import com.astrolabsoftware.spark3d.spatial3DRDD.Point3DRDD
 
 // We assume hdu#1 of filename contains at least 3 columns whose names are `colnames`
 // Order of columns in the file does not matter, as they will be re-aranged
@@ -96,7 +96,7 @@ A sphere is defined by its center (3 spatial coordinates) plus a radius.
 In spark3D, you can choose the coordinate system of the center between cartesian `(x, y, z)` and spherical `(r, theta, phi)`. Let's suppose we have a text file (CSV, JSON, or TXT) whose columns are labeled `r`, `theta`, `phi`, the spherical coordinates and `radius`:
 
 ```scala
-import com.spark3d.spatial3DRDD.SphereRDD
+import com.astrolabsoftware.spark3d.spatial3DRDD.SphereRDD
 
 // We assume filename contains at least 4 columns whose names are `colnames`.
 // Order of columns in the file does not matter, as they will be re-aranged

docs/03_partitioning.md

@@ -25,8 +25,8 @@ There are currently 2 partitioning implemented in the library:
 In the following example, we load `Point3D` data, and we re-partition it with the onion partitioning
 
 ```scala
-import com.spark3d.spatial3DRDD.Point3DRDD
-import com.spark3d.utils.GridType
+import com.astrolabsoftware.spark3d.spatial3DRDD.Point3DRDD
+import com.astrolabsoftware.spark3d.utils.GridType
 
 import org.apache.spark.sql.SparkSession
 
@@ -56,8 +56,8 @@ val pointRDD_partitioned = pointRDD.spatialPartitioning(GridType.LINEARONIONGRID
 In the following example, we load `Point3D` data, and we re-partition it with the octree partitioning
 
 ```scala
-import com.spark3d.spatial3DRDD.Point3DRDD
-import com.spark3d.utils.GridType
+import com.astrolabsoftware.spark3d.spatial3DRDD.Point3DRDD
+import com.astrolabsoftware.spark3d.utils.GridType
 
 import org.apache.spark.sql.SparkSession
 

docs/04_query.md

@@ -14,9 +14,9 @@ The spark3D library contains a number of methods and tools to manipulate 3D RDD.
 A Envelope query takes as input a `RDD[Shape3D]` and an envelope, and returns all objects in the RDD intersecting the envelope (contained in and crossing the envelope):
 
 ```scala
-import com.spark3d.spatial3DRDD.Point3DRDD
-import com.spark3d.geometryObjects.{Point3D, ShellEnvelope}
-import com.spark3d.spatialOperator.RangeQuery
+import com.astrolabsoftware.spark3d.spatial3DRDD.Point3DRDD
+import com.astrolabsoftware.spark3d.geometryObjects.{Point3D, ShellEnvelope}
+import com.astrolabsoftware.spark3d.spatialOperator.RangeQuery
 
 import org.apache.spark.sql.SparkSession
 
@@ -53,7 +53,7 @@ Envelope = Sphere |Envelope = Box
 A cross-match takes as input two data sets, and return objects matching based on the center distance, or pixel index of objects. Note that performing a cross-match between a data set of N elements and another of M elements is a priori a NxM operation - so it can be very costly! Let's load two `Point3D` data sets:
 
 ```scala
-import com.spark3d.spatial3DRDD.Point3DRDD
+import com.astrolabsoftware.spark3d.spatial3DRDD.Point3DRDD
 
 import org.apache.spark.sql.SparkSession
 
@@ -77,8 +77,8 @@ By default, the two sets are partitioned randomly (in the sense points spatially
 In order to decrease the cost of performing the cross-match, you need to partition the two data sets the same way. By doing so, you will cross-match only points belonging to the same partition. For a large number of partitions, you will decrease significantly the cost:
 
 ```scala
-import com.spark3d.utils.GridType
-import com.spark3d.spatialPartitioning.SpatialPartitioner
+import com.astrolabsoftware.spark3d.utils.GridType
+import com.astrolabsoftware.spark3d.spatialPartitioning.SpatialPartitioner
 
 // nPart is the wanted number of partitions. Default is setA_raw partition number.
 // For the spatial partitioning, you can currently choose between LINEARONIONGRID, or OCTREE.
@@ -114,7 +114,7 @@ Currently, we implemented two methods to perform a cross-match:
 Here is an example which returns only elements from B with counterpart in A using distance center:
 
 ```scala
-import com.spark3d.spatialOperator.CenterCrossMatch
+import com.astrolabsoftware.spark3d.spatialOperator.CenterCrossMatch
 
 // Distance threshold for the match
 val epsilon = 0.004
@@ -127,7 +127,7 @@ val xMatchCenter = CenterCrossMatch
 and the same using the Healpix indices:
 
 ```scala
-import com.spark3d.spatialOperator.PixelCrossMatch
+import com.astrolabsoftware.spark3d.spatialOperator.PixelCrossMatch
 
 // Shell resolution for Healpix indexing
 val nside = 512

docs/_pages/home.md

@@ -7,7 +7,7 @@ header:
   cta_label: "<i class='fas fa-download'></i> Install Now"
   cta_url: "/docs/installation/"
   caption:
-excerpt: 'Spark extension for processing large-scale 3D data sets: Astrophysics, High Energy Physics, Meteorology, ...<br /> <small><a href="https://github.com/astrolabsoftware/spark3D/releases/tag/0.1.1">Latest release v0.1.1</a></small><br /><br /> {::nomarkdown}<iframe style="display: inline-block;" src="https://ghbtns.com/github-btn.html?user=astrolabsoftware&repo=spark3D&type=star&count=true&size=large" frameborder="0" scrolling="0" width="160px" height="30px"></iframe> <iframe style="display: inline-block;" src="https://ghbtns.com/github-btn.html?user=astrolabsoftware&repo=spark3D&type=fork&count=true&size=large" frameborder="0" scrolling="0" width="158px" height="30px"></iframe>{:/nomarkdown}'
+excerpt: 'Spark extension for processing large-scale 3D data sets: Astrophysics, High Energy Physics, Meteorology, ...<br /> <small><a href="https://github.com/astrolabsoftware/spark3D/releases/tag/0.1.3">Latest release v0.1.3</a></small><br /><br /> {::nomarkdown}<iframe style="display: inline-block;" src="https://ghbtns.com/github-btn.html?user=astrolabsoftware&repo=spark3D&type=star&count=true&size=large" frameborder="0" scrolling="0" width="160px" height="30px"></iframe> <iframe style="display: inline-block;" src="https://ghbtns.com/github-btn.html?user=astrolabsoftware&repo=spark3D&type=fork&count=true&size=large" frameborder="0" scrolling="0" width="158px" height="30px"></iframe>{:/nomarkdown}'
 feature_row:
   - image_path:
     alt:

examples/jupyter/CrossMatch.ipynb

@@ -154,7 +154,7 @@
    },
    "outputs": [],
    "source": [
-    "import com.spark3d.spatial3DRDD._\n",
+    "import com.astrolabsoftware.spark3d.spatial3DRDD._\n",
     "import org.apache.spark.sql.SparkSession\n",
     "val spark = SparkSession.builder().appName(\"Xmatch\").getOrCreate()\n",
     "\n",
@@ -192,8 +192,8 @@
    },
    "outputs": [],
    "source": [
-    "import com.spark3d.utils.GridType\n",
-    "import com.spark3d.spatialPartitioning.SpatialPartitioner\n",
+    "import com.astrolabsoftware.spark3d.utils.GridType\n",
+    "import com.astrolabsoftware.spark3d.spatialPartitioning.SpatialPartitioner\n",
     "\n",
     "// As we are in local mode, and the file is very small, the RDD pointRDD has only 1 partition.\n",
     "// For the sake of this example, let's increase the number of partition to 100.\n",
@@ -244,7 +244,7 @@
     }
    ],
    "source": [
-    "import com.spark3d.spatialOperator.PixelCrossMatch\n",
+    "import com.astrolabsoftware.spark3d.spatialOperator.PixelCrossMatch\n",
     "\n",
     "// Shell resolution\n",
     "val nside = 512\n",
@@ -298,7 +298,7 @@
     }
    ],
    "source": [
-    "import com.spark3d.spatialOperator.CenterCrossMatch\n",
+    "import com.astrolabsoftware.spark3d.spatialOperator.CenterCrossMatch\n",
     "\n",
     "// Distance threshold for the match\n",
     "val epsilon = 0.004\n",
@@ -336,9 +336,9 @@
     "import javax.swing.JFrame\n",
     "import javax.swing.JPanel\n",
     "\n",
-    "import com.spark3d.utils.Utils.sphericalToCartesian\n",
+    "import com.astrolabsoftware.spark3d.utils.Utils.sphericalToCartesian\n",
     "import org.apache.spark.rdd.RDD\n",
-    "import com.spark3d.geometryObjects._\n",
+    "import com.astrolabsoftware.spark3d.geometryObjects._\n",
     "\n",
     "\n",
     "/** Define palette of colors */\n",

examples/jupyter/onion_partitioning.ipynb

@@ -101,7 +101,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import com.spark3d.spatial3DRDD._\n",
+    "import com.astrolabsoftware.spark3d.spatial3DRDD._\n",
     "import org.apache.spark.sql.SparkSession\n",
     "val spark = SparkSession.builder().appName(\"OnionSpace\").getOrCreate()\n",
     "\n",
@@ -137,7 +137,7 @@
    },
    "outputs": [],
    "source": [
-    "import com.spark3d.utils.GridType\n",
+    "import com.astrolabsoftware.spark3d.utils.GridType\n",
     "\n",
     "// As we are in local mode, and the file is very small, the RDD pointRDD has only 1 partition.\n",
     "// For the sake of this example, let's increase the number of partition to 5.\n",
@@ -201,9 +201,9 @@
     "import javax.swing.JFrame\n",
     "import javax.swing.JPanel\n",
     "\n",
-    "import com.spark3d.utils.Utils.sphericalToCartesian\n",
+    "import com.astrolabsoftware.spark3d.utils.Utils.sphericalToCartesian\n",
     "import org.apache.spark.rdd.RDD\n",
-    "import com.spark3d.geometryObjects._\n",
+    "import com.astrolabsoftware.spark3d.geometryObjects._\n",
     "\n",
     "/** \n",
     "  * Define palette of colors \n",

run_scala.sh

@@ -18,7 +18,7 @@ SBT_VERSION=2.11.8
 SBT_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.1.1
+VERSION=0.1.3
 
 # Package it
 sbt ++${SBT_VERSION} package
@@ -31,7 +31,7 @@ display="show"
 
 ## Dependencies
 jars="lib/jhealpix.jar,lib/swingx-0.9.1.jar"
-packages="com.github.astrolabsoftware:spark-fits_2.11:0.3.0,com.github.haifengl:smile-core:1.5.1,com.github.haifengl:smile-plot:1.5.1,com.github.haifengl:smile-math:1.5.1,com.github.haifengl:smile-scala_2.11:1.5.1"
+packages="com.github.astrolabsoftware:spark-fits_2.11:0.4.0,com.github.haifengl:smile-core:1.5.1,com.github.haifengl:smile-plot:1.5.1,com.github.haifengl:smile-math:1.5.1,com.github.haifengl:smile-scala_2.11:1.5.1"
 
 # Run it!
 spark-submit \

run_xmatch_cluster.sh

@@ -18,7 +18,7 @@ SBT_VERSION=2.11.8
 SBT_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.1.1
+VERSION=0.1.3
 
 # Package it
 sbt ++${SBT_VERSION} package

src/main/scala/com/spark3d/examples/CrossMatch.scala

@@ -13,14 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.spark3d.examples
+package com.astrolabsoftware.spark3d.examples
 
 // spark3d lib
-import com.spark3d.utils.GridType
-import com.spark3d.spatial3DRDD.Point3DRDD
-import com.spark3d.spatialPartitioning.SpatialPartitioner
-import com.spark3d.spatialOperator.PixelCrossMatch
-import com.spark3d.serialization.Spark3dConf.spark3dConf
+import com.astrolabsoftware.spark3d.utils.GridType
+import com.astrolabsoftware.spark3d.spatial3DRDD.Point3DRDD
+import com.astrolabsoftware.spark3d.spatialPartitioning.SpatialPartitioner
+import com.astrolabsoftware.spark3d.spatialOperator.PixelCrossMatch
+import com.astrolabsoftware.spark3d.serialization.Spark3dConf.spark3dConf
 
 // Spark lib
 import org.apache.spark.sql.SparkSession

src/main/scala/com/spark3d/examples/OnionSpace.scala

@@ -13,12 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.spark3d.examples
+package com.astrolabsoftware.spark3d.examples
 
 // spark3d lib
-import com.spark3d.utils.GridType
-import com.spark3d.utils.Utils.sphericalToCartesian
-import com.spark3d.spatial3DRDD.Point3DRDD
+import com.astrolabsoftware.spark3d.utils.GridType
+import com.astrolabsoftware.spark3d.utils.Utils.sphericalToCartesian
+import com.astrolabsoftware.spark3d.spatial3DRDD.Point3DRDD
 
 // Spark lib
 import org.apache.spark.sql.SparkSession

src/main/scala/com/spark3d/geometryObjects/BoxEnvelope.scala

@@ -13,10 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.spark3d.geometryObjects
+package com.astrolabsoftware.spark3d.geometryObjects
 
-import com.spark3d.geometryObjects.Shape3D._
-import com.spark3d.utils.Utils.sphericalToCartesian
+import com.astrolabsoftware.spark3d.geometryObjects.Shape3D._
+import com.astrolabsoftware.spark3d.utils.Utils.sphericalToCartesian
 
 import scala.math._
 

src/main/scala/com/spark3d/geometryObjects/Point3D.scala

@@ -13,10 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.spark3d.geometryObjects
+package com.astrolabsoftware.spark3d.geometryObjects
 
-import com.spark3d.geometryObjects.Shape3D._
-import com.spark3d.utils.Utils.sphericalToCartesian
+import com.astrolabsoftware.spark3d.geometryObjects.Shape3D._
+import com.astrolabsoftware.spark3d.utils.Utils.sphericalToCartesian
 
 /**
   * Class for describing a point in 3D space.

src/main/scala/com/spark3d/geometryObjects/Shape3D.scala

@@ -13,10 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.spark3d.geometryObjects
+package com.astrolabsoftware.spark3d.geometryObjects
 
-import com.spark3d.utils.Utils._
-import com.spark3d.utils.ExtPointing
+import com.astrolabsoftware.spark3d.utils.Utils._
+import com.astrolabsoftware.spark3d.utils.ExtPointing
 
 import healpix.essentials.HealpixBase
 import healpix.essentials.Pointing