pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
      http://www.apache.org/licenses/LICENSE-2.0
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>io.glutenproject</groupId>
  <artifactId>gluten-parent</artifactId>
  <version>1.0.0-SNAPSHOT</version>
  <packaging>pom</packaging>

  <name>Gluten Parent Pom</name>
  <url>https://github.com/oap-project/gluten.git</url>

  <licenses>
    <license>
      <name>The Apache Software License, Version 2.0</name>
      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
    </license>
  </licenses>

  <modules>
    <module>shims</module>
    <module>gluten-core</module>
    <module>package</module>
    <module>substrait/substrait-spark</module>
  </modules>

  <properties>
    <spark32.version>3.2.2</spark32.version>
    <spark32.scala>2.12.15</spark32.scala>
    <spark32bundle.version>3.2</spark32bundle.version>
    <spark33.version>3.3.1</spark33.version>
    <spark33.scala>2.12.15</spark33.scala>
    <spark33bundle.version>3.3</spark33bundle.version>
    <delta20.version>2.0.1</delta20.version>
    <delta22.version>2.2.0</delta22.version>
    <delta.version>${delta20.version}</delta.version>
    <delta.binary.version>20</delta.binary.version>
    <scala.binary.version>2.12</scala.binary.version>
    <scala.version>${spark32.scala}</scala.version>
    <spark.version>${spark32.version}</spark.version>
    <sparkbundle.version>${spark32bundle.version}</sparkbundle.version>
    <arrow.version>10.0.0-SNAPSHOT</arrow.version>
    <arrow-memory.artifact>arrow-memory-unsafe</arrow-memory.artifact>
    <hadoop.version>${hadoop.version}</hadoop.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <project.prefix>spark-sql-columnar</project.prefix>
    <jar.assembly.name.prefix>gluten</jar.assembly.name.prefix>
    <!-- To build built-in backend c++ codes -->
    <scala.recompile.mode>all</scala.recompile.mode>
    <!-- For unit tests -->
    <clickhouse.lib.path>/usr/local/clickhouse/lib/libch.so</clickhouse.lib.path>
    <tpcds.data.path>/data/tpcds-data-sf1</tpcds.data.path>
    <fasterxml.spark33.version>2.13.3</fasterxml.spark33.version>
    <fasterxml.version>${fasterxml.spark33.version}</fasterxml.version>
    <junit.version>4.13.1</junit.version>

    <substrait.version>0.5.0</substrait.version>
    <guava.version>31.1-jre</guava.version>
    <protobuf.version>3.16.3</protobuf.version>
    <protobuf.substrait.version>3.21.7</protobuf.substrait.version>
    <!--spotless-->
    <spotless.version>2.27.2</spotless.version>
    <spotless.scalafmt.version>3.5.9</spotless.scalafmt.version>
    <spotless.delimiter>package</spotless.delimiter>
    <spotless.license.header>
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
    </spotless.license.header>

    <!-- plugin version-->
    <build-helper-maven-plugin.version>3.2.0</build-helper-maven-plugin.version>
    <scala.compiler.version>4.3.0</scala.compiler.version>
    <maven.compiler.plugin>3.8.0</maven.compiler.plugin>
    <maven.jar.plugin>3.2.2</maven.jar.plugin>
    <scalastyle.version>1.0.0</scalastyle.version>
    <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
  </properties>

  <profiles>
    <profile>
      <id>spark-3.2</id>
      <activation>
        <activeByDefault>true</activeByDefault>
      </activation>
      <properties>
        <scala.version>${spark32.scala}</scala.version>
        <spark.version>${spark32.version}</spark.version>
        <delta.version>${delta20.version}</delta.version>
        <delta.binary.version>20</delta.binary.version>
        <sparkbundle.version>${spark32bundle.version}</sparkbundle.version>
      </properties>
    </profile>
    <profile>
      <id>spark-3.3</id>
      <properties>
        <scala.version>${spark33.scala}</scala.version>
        <spark.version>${spark33.version}</spark.version>
        <delta.version>${delta22.version}</delta.version>
        <delta.binary.version>22</delta.binary.version>
        <sparkbundle.version>${spark33bundle.version}</sparkbundle.version>
      </properties>
    </profile>
    <profile>
      <id>hadoop-2.7.4</id>
      <activation>
        <property>
          <name>!hadoop.version</name>
        </property>
      </activation>
      <properties>
        <hadoop.version>2.7.4</hadoop.version>
      </properties>
    </profile>
    <profile>
      <id>hadoop-3.2</id>
      <properties>
        <hadoop.version>3.2.0</hadoop.version>
      </properties>
    </profile>
    <profile>
      <id>dataproc-2.0</id>
      <properties>
        <hadoop.version>3.2.2</hadoop.version>
      </properties>
    </profile>
    <profile>
      <id>emr-6.3.0</id>
      <properties>
        <hadoop.version>3.2.2</hadoop.version>
      </properties>
    </profile>
    <profile>
      <id>arrow-netty</id>
      <activation>
        <activeByDefault>false</activeByDefault>
      </activation>
      <properties>
        <id>arrow-netty</id>
        <arrow-memory.artifact>arrow-memory-netty</arrow-memory.artifact>
      </properties>
    </profile>

    <profile>
      <id>backends-clickhouse</id>
      <activation>
        <activeByDefault>false</activeByDefault>
      </activation>
      <modules>
        <module>backends-clickhouse</module>
      </modules>
      <properties>
        <backend_type>clickhouse</backend_type>
      </properties>
    </profile>
    <profile>
      <id>backends-velox</id>
      <activation>
        <activeByDefault>false</activeByDefault>
      </activation>
      <modules>
        <module>gluten-data</module>
        <module>backends-velox</module>
      </modules>
      <properties>
        <backend_type>velox</backend_type>
        <arrow.version>10.0.0-SNAPSHOT</arrow.version>
        <build_velox_backend>ON</build_velox_backend>
      </properties>
    </profile>
    <profile>
      <id>backends-gazelle</id>
      <activation>
        <activeByDefault>false</activeByDefault>
      </activation>
      <modules>
        <module>gluten-data</module>
        <module>backends-gazelle</module>
      </modules>
      <properties>
        <backend_type>gazelle_cpp</backend_type>
        <arrow.version>8.0.0-gluten-SNAPSHOT</arrow.version>
      </properties>
    </profile>
    <profile>
      <id>spark-ut</id>
      <modules>
        <module>gluten-ut</module>
      </modules>
    </profile>
  </profiles>

  <dependencyManagement>
    <dependencies>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>provided</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>arrow-vector</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>curator-recipes</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <exclusions>
          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.antlr</groupId>
            <artifactId>antlr4-runtime</artifactId>
          </exclusion>
        </exclusions>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>provided</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>curator-recipes</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>com.google.guava</groupId>
        <artifactId>guava</artifactId>
        <version>${guava.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>io.substrait</groupId>
        <artifactId>core</artifactId>
        <version>${substrait.version}</version>
        <exclusions>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-jdk14</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <!-- test dependencies -->
      <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>${junit.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <type>test-jar</type>
        <scope>test</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>curator-recipes</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <exclusions>
          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
        <type>test-jar</type>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <exclusions>
          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
        <type>test-jar</type>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>${hadoop.version}</version>
        <scope>provided</scope>
        <exclusions>
          <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-core</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>io.delta</groupId>
        <artifactId>delta-core_${scala.binary.version}</artifactId>
        <version>${delta.version}</version>
        <scope>provided</scope>
        <exclusions>
          <exclusion>
            <groupId>org.antlr</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest_${scala.binary.version}</artifactId>
        <version>3.2.3</version>
        <scope>test</scope>
      </dependency>
      <!-- Fasterxml -->
      <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-annotations</artifactId>
        <version>${fasterxml.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-core</artifactId>
        <version>${fasterxml.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>${fasterxml.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.datatype</groupId>
        <artifactId>jackson-datatype-guava</artifactId>
        <version>${fasterxml.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.module</groupId>
        <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
        <version>${fasterxml.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-source-plugin</artifactId>
        <version>3.2.1</version>
      </dependency>
    </dependencies>
  </dependencyManagement>

  <build>
    <pluginManagement>
      <plugins>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-jar-plugin</artifactId>
          <version>${maven.jar.plugin}</version>
          <executions>
            <execution>
              <goals>
                <goal>jar</goal>
              </goals>
              <configuration>
                <classifier>${scala.binary.version}</classifier>
                <classifier>${sparkbundle.version}</classifier>
              </configuration>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-compiler-plugin</artifactId>
          <version>${maven.compiler.plugin}</version>
          <configuration>
            <source>1.8</source>
            <target>1.8</target>
            <encoding>UTF-8</encoding>
            <maxmem>1024m</maxmem>
            <fork>true</fork>
            <compilerArgs>
              <arg>-Xlint:all,-serial,-path</arg>
            </compilerArgs>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.scalatest</groupId>
          <artifactId>scalatest-maven-plugin</artifactId>
          <version>${scalatest-maven-plugin.version}</version>
          <configuration>
            <testFailureIgnore>false</testFailureIgnore>
            <junitxml>.</junitxml>
          </configuration>
          <executions>
            <execution>
              <id>test</id>
              <goals>
                <goal>test</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>net.alchim31.maven</groupId>
          <artifactId>scala-maven-plugin</artifactId>
          <version>${scala.compiler.version}</version>
          <configuration>
            <recompileMode>${scala.recompile.mode}</recompileMode>
            <args>
              <arg>-Ywarn-unused:imports</arg>
            </args>
          </configuration>
          <executions>
            <execution>
              <id>scala-compile-first</id>
              <phase>process-resources</phase>
              <goals>
                <goal>add-source</goal>
                <goal>compile</goal>
              </goals>
            </execution>
            <execution>
              <id>scala-test-compile-first</id>
              <phase>process-test-resources</phase>
              <goals>
                <goal>testCompile</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-checkstyle-plugin</artifactId>
          <version>3.1.1</version>
          <configuration>
            <failOnViolation>true</failOnViolation>
            <includeTestSourceDirectory>true</includeTestSourceDirectory>
            <sourceDirectories>
              <directory>${project.basedir}/src/main/java</directory>
              <directory>${project.basedir}/src/main/scala</directory>
            </sourceDirectories>
            <testSourceDirectories>
              <directory>${project.basedir}/src/test/java</directory>
            </testSourceDirectories>
            <configLocation>dev/checkstyle.xml</configLocation>
            <outputFile>${project.basedir}/target/checkstyle-output.xml</outputFile>
            <inputEncoding>UTF-8</inputEncoding>
            <outputEncoding>UTF-8</outputEncoding>
          </configuration>
          <dependencies>
            <dependency>
              <groupId>com.puppycrawl.tools</groupId>
              <artifactId>checkstyle</artifactId>
              <version>8.29</version>
            </dependency>
          </dependencies>
          <executions>
            <execution>
              <phase>validate</phase>
              <goals>
                <goal>check</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.scalastyle</groupId>
          <artifactId>scalastyle-maven-plugin</artifactId>
          <version>${scalastyle.version}</version>
          <configuration>
            <verbose>false</verbose>
            <failOnViolation>true</failOnViolation>
            <includeTestSourceDirectory>true</includeTestSourceDirectory>
            <failOnWarning>false</failOnWarning>
            <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
            <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
            <outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
            <outputEncoding>UTF-8</outputEncoding>
          </configuration>
          <executions>
            <execution>
              <phase>validate</phase>
              <goals>
                <goal>check</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <artifactId>maven-assembly-plugin</artifactId>
          <version>3.3.0</version>
          <configuration>
            <descriptorRefs>
              <descriptorRef>jar-with-dependencies</descriptorRef>
            </descriptorRefs>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-resources-plugin</artifactId>
          <version>3.0.1</version>
        </plugin>
        <plugin>
          <groupId>com.diffplug.spotless</groupId>
          <artifactId>spotless-maven-plugin</artifactId>
          <version>${spotless.version}</version>
          <configuration>
            <!--            <java>-->
            <!--              <googleJavaFormat>-->
            <!--                <version>1.7</version>-->
            <!--                <style>AOSP</style>-->
            <!--              </googleJavaFormat>-->

            <!--              &lt;!&ndash; \# refers to the static imports &ndash;&gt;-->
            <!--              <importOrder>-->
            <!--                <order>io.glutenproject,io.substrait.spark,,javax,java,scala,\#</order>-->
            <!--              </importOrder>-->

            <!--              <removeUnusedImports />-->
            <!--            </java>-->
            <scala>
              <scalafmt>
                <version>${spotless.scalafmt.version}</version>
                <scalaMajorVersion>${scala.binary.version}</scalaMajorVersion>
                <file>.scalafmt.conf</file>
              </scalafmt>
              <licenseHeader>
                <content>${spotless.license.header}</content>
                <delimiter>${spotless.delimiter}</delimiter>
              </licenseHeader>
            </scala>
          </configuration>
          <executions>
            <execution>
              <id>spotless-check</id>
              <phase>validate</phase>
              <goals>
                <goal>check</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.codehaus.mojo</groupId>
          <artifactId>build-helper-maven-plugin</artifactId>
          <version>${build-helper-maven-plugin.version}</version>
        </plugin>
      </plugins>
    </pluginManagement>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-enforcer-plugin</artifactId>
        <version>1.0.1</version>
        <executions>
          <execution>
            <id>enforce-versions</id>
            <goals>
              <goal>enforce</goal>
            </goals>
            <configuration>
              <rules>
                <requireMavenVersion>
                  <version>3.6.3</version>
                </requireMavenVersion>
              </rules>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-source-plugin</artifactId>
        <version>3.2.1</version>
        <executions>
          <execution>
            <id>attach-sources</id>
            <phase>verify</phase>
            <goals>
              <goal>jar-no-fork</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>
</project>