build.xml

<project name="datafu" basedir="." default="all"
         xmlns:ivy="antlib:org.apache.ivy.ant"
         xmlns:artifact="antlib:org.apache.maven.artifact.ant">
  <property file="build.properties" />

  <property name="name" value="datafu" />
  <property name="display.name" value="DataFu" />
  <property name="author" value="Matthew Hayes, Sam Shah" />
  <property environment="env" />
  <property name="datafu.version" value="0.0.11" />
  <property name="version" value="${datafu.version}-SNAPSHOT" />
  <property name="final.name" value="${name}-${version}" />
  
  <!-- maven snapshots and staging repository id and url -->
  <property name="maven-snapshots-repository-id" value="sonatype-nexus-snapshots" />
  <property name="maven-snapshots-repository-url" value="https://oss.sonatype.org/content/repositories/snapshots" />
  <property name="maven-staging-repository-id" value="sonatype-nexus-staging" />
  <property name="maven-staging-repository-url" value="https://oss.sonatype.org/service/local/staging/deploy/maven2" />
  
  <!-- directories -->
  <property name="src.dir" value="${basedir}/src" />
  <property name="java.dir" value="${src.dir}/java" />
  <property name="plugin.java.dir" value="${basedir}/plugin/java" />
  <property name="test.dir" value="${basedir}/test" />
  <property name="pigtestsrc.dir" value="${test.dir}/pig" />
  <property name="dist.dir" value="${basedir}/dist" />
  <property name="tools.dir" value="${basedir}/tools" />
  <property name="lib.dir" value="${basedir}/lib" />
  <property name="common.lib.dir" value="${lib.dir}/common" />
  <property name="tools.lib.dir" value="${lib.dir}/tools" />
  <property name="static.lib.dir" value="${basedir}/staticlibs" />
  <property name="classes.dir" value="${dist.dir}/classes" />
  <property name="plugin.classes.dir" value="${dist.dir}/pluginclasses" />
  <property name="report.dir" value="${basedir}/report" />
  <property name="report.pig.dir" value="${report.dir}/pig" />
  <property name="docs.dir" value="${basedir}/docs" />
  <property name="javadoc.dir" value="${docs.dir}/javadoc" />
  <property name="cobertura.dir" value="${basedir}/cobertura" />
  <property name="coveragereport.dir" value="${basedir}/coverage-report" />
  <property name="instrumented.dir" value="${basedir}/instrumented" />
  <property name="pigtestclasses.dir" value="${dist.dir}/pigtestclasses" />
  <property name="fastutil.jar" value="fastutil-jar-6.3.jar" />
  
  <!-- output names -->
  <property name="output.jarfile" value="${dist.dir}/${final.name}.jar" />
  <property name="output.jarfile.sources" value="${dist.dir}/${final.name}-sources.jar" />
  <property name="output.jarfile.javadoc" value="${dist.dir}/${final.name}-javadoc.jar" />
  
  <property name="testclasses.pattern" value="**/*.class" />
      
  <property name="ivy.jar.dir" value="${basedir}/ivy" />
  <property name="datafu.pom.name" value="pom.xml" />
  <property name="datafu.pomfile" value="${ivy.jar.dir}/${datafu.pom.name}" />
  <property name="datafu.pomfile.template" value="${ivy.jar.dir}/pom-template.xml" />
  <property name="ivy.jar.version" value="2.2.0" />
  <property name="maven.jar.version" value="2.0.8" />
  <property name="mvnrepo" value="http://repo2.maven.org/maven2"/>
  <property name="maven.jar.repo.url" value="${mvnrepo}/org/apache/maven/maven-ant-tasks/${maven.jar.version}/maven-ant-tasks-${maven.jar.version}.jar"/>
  <property name="ivy.jar.repo.url" value="${mvnrepo}/org/apache/ivy/ivy/${ivy.jar.version}/ivy-${ivy.jar.version}.jar"/>
	<property name="ivy.jar" value="${ivy.jar.dir}/ivy-${ivy.jar.version}.jar" />
	<property name="ivy.settings.file" value="ivysettings.xml" />
	<property name="maven.jar" location="${ivy.jar.dir}/maven-ant-tasks-${maven.jar.version}.jar"/>
  
  <loadproperties srcfile="${ivy.jar.dir}/libraries.properties"/>

  <basename property="basedirname" file="${basedir}"/>
  
  <target name="all" depends="clean, jar, sources-jar, javadoc-jar" description="Build all artifacts." />
    
  <target name="init" depends="ivy-resolve">
    <mkdir dir="${dist.dir}" />
  </target>

  <target name="eclipse" depends="jar,jar-plugin">
    <delete file="${basedir}/.factorypath"/>
    <copy file="${basedir}/.factorypath.template"
             tofile="${basedir}/.factorypath">
        <filterchain>
           <replacetokens>
                <token key="HOME" value="/${basedirname}"/>
            </replacetokens>
        </filterchain>
    </copy>
    <echo message="Created .factorypath"/>
    <delete file="${basedir}/.classpath"/>
    <copy file="${basedir}/.classpath.template"
             tofile="${basedir}/.classpath">
        <filterchain>
           <replacetokens>
                <token key="PROJECT" value="${basedirname}"/>
            </replacetokens>
        </filterchain>
    </copy>
    <echo message="Created .classpath"/>
  </target>
    
  <target name="maven-ant-tasks-jar-download" description="Download maven tasks jar">
    <mkdir dir="${ivy.jar.dir}"/>
    <get src="${maven.jar.repo.url}" dest="${maven.jar}" usetimestamp="true"/>
  </target>
  
  <target name="maven-taskdef" depends="maven-ant-tasks-jar-download">
    <path id="mvn-ant-task.classpath" path="${maven.jar}"/>
    <typedef resource="org/apache/maven/artifact/ant/antlib.xml"
        uri="urn:maven-artifact-ant"
        classpathref="mvn-ant-task.classpath"/>
  </target>
  
  <target name="ivy-jar-download" description="Download ivy jar">
    <mkdir dir="${ivy.jar.dir}"/>
    <get src="${ivy.jar.repo.url}" dest="${ivy.jar}" usetimestamp="true"/>
  </target>
    
  <target name="ivy-taskdef" depends="ivy-jar-download" description="install ivy">
   	<path id="ivy.lib.path">
   	  <pathelement location="${ivy.jar}"/>
   	</path>
   	<taskdef resource="org/apache/ivy/ant/antlib.xml"
             uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path"/>
  </target>
  
  <target name="ivy-resolve" depends="ivy-taskdef" description="retrieve dependencies with ivy">
    <ivy:retrieve pattern="${lib.dir}/[conf]/[artifact]-[type]-[revision].[ext]"/>
  </target>

  <target name="check-licenses" description="checks that source files have license header">
    <exec executable="./check-license-headers.sh" failonerror="true"/>
  </target>
  
  <target name="clean-cache" depends="ivy-taskdef" description="clean the ivy cache">
  	<ivy:cleancache />
  </target>
  
  <path id="main-classpath">
    <fileset dir="${common.lib.dir}">
      <include name="*.jar" />
    </fileset>
    <pathelement path="${classes.dir}" />
  </path>  
  
  <path id="cobertura.classpath">
    <fileset dir="${cobertura.dir}">
      <include name="cobertura.jar" />
      <include name="lib/**/*.jar" />
    </fileset>
  </path>
  
  <taskdef classpathref="cobertura.classpath" resource="tasks.properties" />
  
  <path id="plugin-classpath">
    <fileset dir="${static.lib.dir}">
      <include name="*.jar" />
    </fileset>
    <fileset dir="${tools.lib.dir}">
      <include name="*.jar" />
    </fileset>
  </path>

  <path id="base-test-classpath">    
    <fileset dir="${common.lib.dir}">
      <include name="*.jar" />
    </fileset>
    <fileset dir="${static.lib.dir}">
      <include name="*.jar" />
    </fileset>
    <pathelement path="${classes.dir}" />
    <pathelement path="${pigtestclasses.dir}" />
  </path>
  
  <path id="test-classpath">
    <path refid="base-test-classpath" />
    <!-- Use non-instrumented JARs for normal testing, otherwise it is 15% slower to run the tests -->
    <fileset dir="${dist.dir}">
      <include name="*.jar" />
    </fileset>
    <pathelement path="${plugin.classes.dir}" />
  </path>
  
  <path id="instrumented-test-classpath">
    <path refid="base-test-classpath" />
    <path refid="cobertura.classpath" />
    <!-- Use instrumented JARs -->
    <fileset dir="${instrumented.dir}">
      <include name="*.jar" />
    </fileset>
  </path>
  
  <macrodef name="replace-dir">
    <attribute name="dir" />
    <sequential>
      <delete dir="@{dir}" />
      <mkdir dir="@{dir}" />
    </sequential>
  </macrodef>
  
  <target name="clean" description="Delete generated files.">
    <delete dir="${basedir}/build" />
    <delete dir="${lib.dir}" />
    <delete dir="${dist.dir}" />
    <delete dir="${docs.dir}" />
    <delete dir="${report.dir}" />
    <delete dir="${coveragereport.dir}"/>
    <delete dir="${instrumented.dir}"/>
    <delete dir="test-output"/>
    <delete file="cobertura.ser"/>
  </target>
  
  <target name="build" depends="init" description="Compile main source tree java files">
    <replace-dir dir="${classes.dir}" />
    <javac destdir="${classes.dir}" target="1.5" debug="true" deprecation="false" failonerror="true" includeantruntime="false">
      <src path="${java.dir}" />
      <exclude name="datafu/linkedin/**"/>
      <classpath refid="main-classpath" />
    </javac>
  </target>

  <target name="build-plugins" depends="init" description="Compile main source tree java files">
    <replace-dir dir="${plugin.classes.dir}" />
    <javac destdir="${plugin.classes.dir}" target="1.5" debug="true" deprecation="false" failonerror="true" includeantruntime="false">
      <src path="${plugin.java.dir}" />
      <classpath refid="plugin-classpath" />
    </javac>
  </target>

  <target name="jar-plugin" depends="build-plugins" description="Builds the plugin JAR">
    <delete file="${basedir}/annotation-plugin.jar" />
    <jar destfile="${basedir}/annotation-plugin.jar">
      <fileset dir="${plugin.classes.dir}">
        <include name="**/*.*" />
      </fileset>
      <service type="javax.annotation.processing.Processor" provider="org.adrianwalker.multilinestring.MultilineProcessor"/>
    </jar>
  </target>
  
  <target name="build-pig-tests" depends="build,build-plugins" description="Compile pig test classes">
    <replace-dir dir="${pigtestclasses.dir}" />
    <copy todir="${pigtestclasses.dir}">
      <fileset dir="${pigtestsrc.dir}">
        <exclude name="**/*.java" />
      </fileset>
    </copy>
    <javac destdir="${pigtestclasses.dir}" target="1.5" debug="true" deprecation="false" failonerror="true" includeantruntime="false">
      <src path="${pigtestsrc.dir}" />
      <compilerarg line="-processor org.adrianwalker.multilinestring.MultilineProcessor"/>
      <classpath refid="test-classpath" />
    </javac>
  </target>
    
  <target name="jar" depends="build" description="Builds the final JAR">
    <!-- build the core jar -->
    <delete file="${dist.dir}/${final.name}.jar" />
    <jar destfile="${dist.dir}/${final.name}.jar">
      <fileset dir="${classes.dir}">
        <include name="**/*.*" />
      </fileset>
    </jar>

    <!-- Need to check for the existence of the jar because next command won't fail if it's missing -->
    <fail message="Could not find fastutil jar">
      <condition>
          <not>
              <resourcecount count="1">
                  <fileset id="fs" dir="${common.lib.dir}" includes="${fastutil.jar}"/>
              </resourcecount>
          </not>
      </condition>
    </fail>
    
    <!-- add fastutil classese to the core jar -->
    <delete file="${dist.dir}/${final.name}-orig.jar" />
    <move file="${dist.dir}/${final.name}.jar" tofile="${dist.dir}/${final.name}-orig.jar" />
    <java jar="${tools.dir}/autojar.jar" fork="true">
      <arg line="-baeq -o ${dist.dir}/${final.name}.jar -c ${common.lib.dir}/${fastutil.jar} ${dist.dir}/${final.name}-orig.jar" />
    </java>
    <delete file="${dist.dir}/${final.name}-orig.jar" />
    
    <!-- build the final jar -->
    <delete file="${dist.dir}/${final.name}-orig.jar" />
    <move file="${dist.dir}/${final.name}.jar" tofile="${dist.dir}/${final.name}-orig.jar" />
    <taskdef name="jarjar" classname="com.tonicsystems.jarjar.JarJarTask" classpath="tools/jarjar-1.1.jar"/>
    <jarjar jarfile="${dist.dir}/${final.name}.jar">
        <zipfileset src="${dist.dir}/${final.name}-orig.jar"/>
        <rule pattern="it.unimi.dsi.fastutil.**" result="datafu.it.unimi.dsi.fastutil.@1"/>
    </jarjar>
    <delete file="${dist.dir}/${final.name}-orig.jar" />
  </target>
    
  <target name="jar-instrumented" depends="jar" description="Build an instrumented jar">
    <delete file="cobertura.ser" />
    <replace-dir dir="${instrumented.dir}" />
    <cobertura-instrument todir="${instrumented.dir}">
      <includeClasses regex=".*datafu.*" />
      <excludeClasses regex=".*fastutil.*" />
      <excludeClasses regex=".*test.*" />
      <!-- Code coverage on LinkedIn specific code is not a priority right now -->
      <excludeClasses regex=".*linkedin.*"/>
      <instrumentationClasspath>
        <pathelement location="${dist.dir}"/>
      </instrumentationClasspath>
    </cobertura-instrument>
    <taskdef name="jarjar" classname="com.tonicsystems.jarjar.JarJarTask" classpath="tools/jarjar-1.1.jar"/>
    <delete file="${instrumented.dir}/${final.name}.jar" />
    <!-- Classes are now instrumented, but we need to merge back with original JAR to get back what was lost, since
         we don't instrument everything and therefore classes are missing form the instrumented directory. -->
    <jarjar jarfile="${instrumented.dir}/${final.name}.jar">      
      <zipfileset src="${dist.dir}/${final.name}.jar"/>        
      <!-- Classes in instrumented directory come last so they take precedence over the classes in the JAR.
           We need to include the JAR so we get the fastutil classes.  The fastutil classes are not instrumented (by design),
           so they are not in the instrumented directory. -->
      <fileset dir="${instrumented.dir}" />
    </jarjar>
  </target>
  
  <target name="test" depends="build-pig-tests, jar" description="Runs the pig tests">
    <taskdef resource="testngtasks" classpath="${common.lib.dir}/testng-jar-6.2.jar"/>
    <testng classpathref="test-classpath"
            outputDir="${report.dir}" verbose="2" haltonfailure="true" haltonskipped="true">
      <jvmarg value="-Xmx1G" />
      <sysproperty key="datafu.jar.dir" value="${dist.dir}" />
      <classfileset dir="${pigtestclasses.dir}" includes="${testclasses.pattern}" />
    </testng>
  </target>
  
  <target name="test-instrumented" depends="build-pig-tests, jar-instrumented" description="Runs the tests with instrumented JARs">
    <taskdef resource="testngtasks" classpath="${common.lib.dir}/testng-6.2.jar"/>
    <testng classpathref="instrumented-test-classpath"
            outputDir="${report.dir}" haltonfailure="true" haltonskipped="true">
      <jvmarg value="-Xmx1G" />
      <sysproperty key="datafu.jar.dir" value="${instrumented.dir}" />
      <classfileset dir="${pigtestclasses.dir}" includes="${testclasses.pattern}" />
    </testng>
  </target>
  
  <target name="coverage" depends="test-instrumented" description="Generates the code coverage report">
    <replace-dir dir="${coveragereport.dir}" />
    <cobertura-report format="html" destdir="${coveragereport.dir}">
      <fileset dir="${java.dir}">
        <include name="**/*.java" />
      </fileset>
    </cobertura-report>
  </target>
  
  <target name="sources-jar" description="Build source jar file">
    <jar destfile="${output.jarfile.sources}">
      <fileset dir="${java.dir}">
        <include name="**/*.java" />
      </fileset>
    </jar>
  </target>
  
  <macrodef name="make-javadocs">
    <attribute name="dir" />
    <attribute name="packagenames" />
    <attribute name="javadir" />
    <sequential>
      <replace-dir dir="@{dir}" />
      <javadoc destdir="@{dir}" windowtitle="${display.name} ${datafu.version}" source="1.5" author="true" version="true" use="true" packagenames="@{packagenames}">
        <doctitle>${display.name} ${datafu.version}</doctitle>
        <bottom>${author}</bottom>
        <classpath refid="main-classpath" />
        <fileset dir="@{javadir}">
          <exclude name="datafu/pig/stats/*?VAR.java"/>
          <exclude name="datafu/pig/sets/SetOperationsBase.java"/>
        </fileset>
      </javadoc>
    </sequential>
  </macrodef>
  
  <target name="javadoc-jar" description="Create complete Javadoc documentation">
    <delete dir="${javadoc.dir}" />
    <make-javadocs dir="${javadoc.dir}" packagenames="*" javadir="${java.dir}"/>
    
    <!-- Make javadoc JAR -->    
    <jar destfile="${output.jarfile.javadoc}">
      <fileset dir="${javadoc.dir}">
        <include name="**/*.*" />
      </fileset>
    </jar>
  </target>
     
  <target name="deploy" depends="clean,jar,sources-jar,javadoc-jar,set-version,sign">
    <typedef resource="org/apache/maven/artifact/ant/antlib.xml"
         uri="antlib:org.apache.maven.artifact.ant"
         classpathref="main-classpath" />

    <artifact:pom file="${datafu.pomfile}" id="datafu-pom"/>
    <artifact:install-provider artifactId="wagon-ssh" version="${wagon-http.version}"/>
    <artifact:deploy file="${output.jarfile}">
      <remoteRepository id="${maven-staging-repository-id}" url="${maven-staging-repository-url}"/>
      <pom refid="datafu-pom"/>
      <attach file="${output.jarfile}.asc" type="jar.asc"/>
      <attach file="${datafu.pomfile}.asc" type="pom.asc"/>
      <attach file="${output.jarfile.sources}.asc" type="jar.asc" classifier="sources" />
      <attach file="${output.jarfile.sources}" classifier="sources" />
    	<attach file="${output.jarfile.javadoc}.asc" type="jar.asc" classifier="javadoc" />
    	<attach file="${output.jarfile.javadoc}" classifier="javadoc" />
    </artifact:deploy>
 </target>
 
 <target name="sign" depends="clean-sign">
   <macrodef name="sign-artifact" description="Signs the artifact">
     <attribute name="input.file"/>
     <attribute name="output.file" default="@{input.file}.asc"/>
     <sequential>
       <echo>Signing @{input.file} Sig File: @{output.file}</echo>
       <exec executable="gpg" >
         <arg value="--armor"/>
         <arg value="--output"/>
         <arg value="@{output.file}"/>
         <arg value="--detach-sig"/>
         <arg value="@{input.file}"/>
       </exec>
     </sequential>
   </macrodef>
   <sign-artifact input.file="${output.jarfile}"/>
   <sign-artifact input.file="${output.jarfile.sources}"/>
   <sign-artifact input.file="${output.jarfile.javadoc}"/>
   <sign-artifact input.file="${datafu.pomfile}"/>
  </target>

  <target name="set-version">
    <delete file="${datafu.pomfile}"/>
    <copy file="${datafu.pomfile.template}" tofile="${datafu.pomfile}"/>
    <replaceregexp byline="true">
      <regexp pattern="@version"/>
      <substitution expression="${datafu.version}"/>
      <fileset dir="${ivy.jar.dir}">
        <include name="${datafu.pom.name}"/>
      </fileset>
    </replaceregexp>
  </target>

  <target name="clean-sign" description="Clean.  Delete .asc files">
    <delete>
      <fileset dir="." includes="**/**/*.asc"/>
    </delete>
  </target>
</project>