build.gradle

plugins {
  id "ru.vyarus.use-python" version "2.3.0"
  id "com.palantir.docker" version "0.32.0"
  id 'com.palantir.docker-run' version "0.32.0"
}

group = ''
description = """NLP Insights"""


ext {
  buildVenv = "${buildDir}/venv"
  rootPackage = "nlp_insights"
  dockerZipFile = "${buildDir}/dockerFileArchive.zip"

  production_pkgs = [
    'ibm-whcs-sdk:1.9.0',
    'fhir.resources:6.1.0',
    'flask:2.1.1'
  ]

  dev_pkgs = [
    'pytest:6.2.5',
    'pytest-cov:3.0.0',
    'deepdiff:5.5.0',
    'flake8:4.0.1',
    'types-requests:2.25.9',
    /* https://docs.python.org/3/library/venv.html
     Common installation tools such as setuptools and pip work as expected with virtual environments.
     In other words, when a virtual environment is active, they install Python packages into the virtual environment
     without needing to be told to do so explicitly.
     Explicitly trying to install wheel causes this task to always run, because wheel is already there,
     but wasn't installed by the build process.
     */
    //'wheel:0.36.2',
    'pytest-html:3.1.1',
    'pylint:2.10.2',
    'pylint2junit:1.0.1',
    'flake8-formatter-junit-xml:0.0.6',
    'mypy:0.931'
  ]
}



python {
  pip production_pkgs
  minPythonVersion = project.minPythonVersion
  minPipVersion = project.minPipVersion
  envPath = buildVenv
  pythonBinary = 'python3'
}


task clean {
  description 'delete build directory'
  group 'Build Setup'
  doLast {
    exec {
      commandLine 'rm', '-rf', project.buildDir
    }
  }
}


task pipForBuild(type: PipInstallTask) {
  finalizedBy "createRequirementsTextForDev"
  description 'Installs required modules for building, linting, and running tests'
  group 'Python'
  pip(dev_pkgs)
}


task createRequirementsTextForDev {
  // intentionally going into the project directory
  // developers need a way to install the required dependencies into their own virtual environment
  // this file can be checked in, it will not be removed on a clean

  inputs.property("prod_modules", production_pkgs)
  inputs.property("dev_modules", dev_pkgs)
  outputs.file(layout.projectDirectory.file('requirements_dev.txt'))

  doLast{
    def pinned_text = new StringBuilder()
    pinned_text.append("### DO NOT CHANGE THIS FILE\n")
    pinned_text.append("### It is generated by the build process: ./gradlew createRequirementsTextForDev\n")
    pinned_text.append("### build.gradle defines the dependencies needed for development and production.\n")
    pinned_text.append("### This file is generated in the SOURCE directory, so that an IDE can be setup for development.\n")
    pinned_text.append("### Using this file, you can install all required dependencies for development into your IDE's venv.\n")
    pinned_text.append("### The pinned.txt that is created in the build directory has only the dependencies needed for production.\n")
    pinned_text.append("### This file may be safely checked into version control so that others may set-up their environments.\n")
    pinned_text.append("### Use this file with \"pip install -r requirements_dev.txt\" to install dependencies into your venv.\n")
    pinned_text.append("### Most IDEs will allow you to specify the file when setting up the venv, \n")
    pinned_text.append("### and you won't have to run that command directly.\n")
    pinned_text.append("### DO NOT CHANGE THIS FILE\n")

    pinned_text.append("\n### Production Dependencies\n")
    production_pkgs.each { module ->
      pinned_text.append(module.replace(":", "==") + "\n")
    }

    pinned_text.append("\n### Development Dependencies\n")
    dev_pkgs.each { module ->
      pinned_text.append(module.replace(":", "==") + "\n")
    }

    outputs.files.singleFile.text = pinned_text.toString()
  }

}


task generateReqs {
  description 'creates pinned.txt file, the additional required modules to run the application'
  outputs.file(layout.buildDirectory.file('pinned.txt')).withPropertyName("pinned")
  inputs.property "prod_deps", production_pkgs.join(",")

  doLast {
    def pinned_text = new StringBuilder()
    production_pkgs.each { module ->
      pinned_text.append(module.replace(":", "==") + "\n")
    }
    outputs.files.singleFile.text = pinned_text.toString()
  }
}


task setupBuildProperties {
  description 'creates setup.properties, which is used by setup.py'
  inputs.property 'projectName', project.name
  inputs.property 'projectVersion', project.version
  inputs.property 'pySrcDir', pySrcDir
  inputs.property 'rootPackage', project.rootPackage
  outputs.file(layout.buildDirectory.file('setup.properties')).withPropertyName("setup")

  doLast {
    outputs.files.singleFile.text = """[default]
                                        |project_name=${inputs.properties['projectName']}
                                        |project_version=${inputs.properties['projectVersion']}
                                        |project_srcDir=${inputs.properties['pySrcDir']}/py
        """.stripMargin()
  }
}


task copySource {
  description 'copies the source code into build directory'
  def srcRoot = (new File("${rootDir}")).absoluteFile.toURI()
  def destRoot = new File("${buildDir}")
  def sourceTree = fileTree("${rootDir}/${pySrcDir}").include("**/*.py")
  def dests = new HashMap<File, File>()

  inputs.files(sourceTree.files)

  sourceTree.files.each { sFile ->
    def relPath = srcRoot.relativize(sFile.absoluteFile.toURI())

    def destFile = new File(destRoot, relPath.toString())
    dests.put(sFile, destFile)
    outputs.file(destFile)

  }

  doLast {
    delete "${buildDir}/${pySrcDir}"
    sourceTree.files.each { sourceFile ->
      copy {
        from sourceFile.absolutePath
        into dests[sourceFile].parent
      }

    }
  }
}


task copyTestResources {
  description 'copies the test source and resources to the build directory'
  def srcRoot = (new File("${rootDir}")).absoluteFile.toURI()
  def destRoot = new File("${buildDir}")
  def sourceTree = fileTree("${rootDir}/${pyTestDir}").include([
    "**/*.py",
    "**/*.json",
    "**/*.txt",
    "**/*.xml"
  ])
  def dests = new HashMap<File, File>()

  inputs.files(sourceTree.files)

  sourceTree.files.each { sFile ->
    def relPath = srcRoot.relativize(sFile.absoluteFile.toURI())

    def destFile = new File(destRoot, relPath.toString())
    dests.put(sFile, destFile)
    outputs.file(destFile)

  }

  doLast {
    delete "${buildDir}/${pyTestDir}"
    sourceTree.files.each { sourceFile ->
      copy {
        from sourceFile.absolutePath
        into dests[sourceFile].parent
      }

    }
  }

}


task buildConfiguration {
  description 'copies setup scripts and configuration to build directory'
  inputs.file("${rootDir}/build_tools/setup.py")
  inputs.file("${rootDir}/setup.cfg")
  outputs.file("${buildDir}/setup.py")
  outputs.file("${buildDir}/setup.cfg")

  doLast {
    copy {
      from "${rootDir}/build_tools"
      include "setup.py"
      into "${buildDir}"
    }

    copy {
      from "${rootDir}"
      include "setup.cfg"
      into "${buildDir}"
    }
  }
}


task build(type: PythonTask) {
  description 'builds a python distribution from the source code'
  dependsOn project.tasks.pipForBuild
  dependsOn project.tasks.buildConfiguration
  dependsOn project.tasks.generateReqs
  dependsOn project.tasks.setupBuildProperties
  dependsOn project.tasks.copySource

  inputs.files(project.tasks.copySource.outputs.files)
  inputs.files(project.tasks.setupBuildProperties.outputs.files)
  inputs.files(project.tasks.generateReqs.outputs.files)
  inputs.property 'minPythonVersion', minPythonVersion

  outputs.file("${buildDir}/${pyDistDir}/${project.name.replace("-", "_")}-${version}-py3-none-any.whl")

  doFirst {
    // need to make sure any deleted files in the source don't stick around
    // in the build lib for the wheel, the wheel build artificats aren't
    // needed outside of the build process, and just create confusion anyway
    delete "${buildDir}/build"
    delete "${buildDir}/${pyDistDir}"
    file("${buildDir}/${pyDistDir}").mkdirs()
  }
  workDir = "${project.buildDir}"
  command = "setup.py"
  extraArgs = [
    'bdist_wheel',
    '-d',
    pyDistDir,
    '-k'
  ]
  doLast {
    delete "${buildDir}/build"
  }
}


task install(type: PythonTask) {
  description 'finializes build by installing the wheel to the virtual env'
  group 'development'
  dependsOn project.tasks.build
  inputs.file(project.tasks.build.outputs.files.singleFile)
  outputs.file("${buildDir}/${project.name}_pip.log")

  workDir = "${project.buildDir}"
  module "pip"
  extraArgs = [
    "install",
    "${project.tasks.build.outputs.files.singleFile.absolutePath}",
    "--force-reinstall",
    "--no-dependencies",
    "--log",
    "${buildDir}/${project.name}_pip.log"
  ]
}


task reportsDir {
  description 'creates the report directory'
  outputs.dir("${buildDir}/${reportsDir}")
  doLast {
    mkdir "${buildDir}/${reportsDir}"
  }
}



task pytest(type: PythonTask) {
  description 'runs testcases'
  group null
  dependsOn project.tasks.pipForBuild
  dependsOn project.tasks.install

  inputs.files(project.tasks.copyTestResources.outputs.files)
  inputs.files(project.tasks.copySource.outputs.files)
  inputs.files(project.tasks.install.outputs.files.singleFile)
  inputs.files(project.tasks.buildConfiguration.outputs.files)

  def pyTestXmlReportFile =  "${project.tasks.reportsDir.outputs.files.singleFile.absolutePath}/pytest-report.xml"
  def coverageDirectory = "${project.tasks.reportsDir.outputs.files.singleFile.absolutePath}/coverage"


  outputs.file(pyTestXmlReportFile)
  outputs.dir(coverageDirectory)
  outputs.file("${buildDir}/.coverage")

  workDir = "${project.buildDir}"
  module = 'pytest'
  extraArgs = [
    "${buildDir}/${pyTestDir}/py",
    "--pyargs",
    "--cov-report=html:${coverageDirectory}",
    "--cov=nlp_insights",
    "--junit-xml=${pyTestXmlReportFile}"
    ]
}


task docTestSource(type: PythonTask) {
  description 'tests documentation of source modules'
  dependsOn project.tasks.pipForBuild
  dependsOn project.tasks.install

  inputs.files(project.tasks.copyTestResources.outputs.files)
  inputs.files(project.tasks.copySource.outputs.files)
  inputs.files(project.tasks.install.outputs.files.singleFile)

  def pyDocTestXmlFile = "${project.tasks.reportsDir.outputs.files.singleFile.absolutePath}/doc-test.xml"
  outputs.file(layout.buildDirectory.file(pyDocTestXmlFile))

  workDir = "${project.buildDir}"
  module = 'pytest'
  extraArgs = [
    "${pySrcDir}/py",
    "--doctest-modules",
    "--junit-xml=${pyDocTestXmlFile}"
  ]

}


task test {
  description 'runs unit and doc tests and places reports in reports directory'
  group 'quality'
  dependsOn project.tasks.pytest
  dependsOn project.tasks.docTestSource
}


task flake8(type: Exec) {
  description 'flake8 linter'
  dependsOn project.tasks.pipForBuild
  dependsOn project.tasks.install
  dependsOn project.tasks.buildConfiguration

  def flake8Xml = "${project.tasks.reportsDir.outputs.files.singleFile.absolutePath}/flake8.xml"
  outputs.file(flake8Xml)

  inputs.files(project.tasks.copySource.outputs.files)
  inputs.files(project.tasks.buildConfiguration.outputs.files)

  standardOutput = new ByteArrayOutputStream()
  ignoreExitValue=true
  workingDir = "${project.buildDir}"
  commandLine "${buildVenv}/bin/flake8", "${pySrcDir}/py", "--format=junit-xml", "--config=setup.cfg"

  doLast {
    outputs.files.singleFile.text = standardOutput.toString()
  }

}


task pylint(type: Exec) {
  description 'pylint linter'
  dependsOn project.tasks.pipForBuild
  dependsOn project.tasks.install
  dependsOn project.tasks.buildConfiguration

  def pylintXml = "${project.tasks.reportsDir.outputs.files.singleFile.absolutePath}/pylint.xml"
  outputs.file(pylintXml)

  inputs.files(project.tasks.copySource.outputs.files)
  inputs.files(project.tasks.buildConfiguration.outputs.files)

  standardOutput = new ByteArrayOutputStream()
  ignoreExitValue=true
  workingDir = "${project.buildDir}"
  commandLine "${buildVenv}/bin/pylint", "${pySrcDir}/py/${rootPackage}", "--output-format=pylint2junit.JunitReporter", "--rcfile=setup.cfg"
  doLast {
    outputs.files.singleFile.text = standardOutput.toString()
  }
}


task mypy(type: Exec) {
  description 'mypy linter'
  dependsOn project.tasks.pipForBuild
  dependsOn project.tasks.install
  dependsOn project.tasks.buildConfiguration

  def myPyXml = "${project.tasks.reportsDir.outputs.files.singleFile.absolutePath}/mypy.xml"

  outputs.file(myPyXml)
  inputs.files(project.tasks.copySource.outputs.files)
  inputs.files(project.tasks.buildConfiguration.outputs.files)

  standardOutput = new ByteArrayOutputStream()
  ignoreExitValue=true
  workingDir = "${project.buildDir}"
  commandLine "${buildVenv}/bin/mypy",
      "${pySrcDir}/py/${rootPackage}",
      "--junit-xml=${myPyXml}",
      "--config-file=${project.buildDir}/setup.cfg"
}


task lint {
  description 'lint checks source code'
  group 'quality'
  dependsOn project.tasks.flake8
  dependsOn project.tasks.pylint
  dependsOn project.tasks.mypy
}


task checkSource {
  description 'prints a summary of potential problems found during the build'
  group 'quality'
  dependsOn project.tasks.lint
  dependsOn project.tasks.test


  doLast {
    groovy.xml.XmlSlurper slurper = new groovy.xml.XmlSlurper();

    def xmlReports = (new File("${buildDir}/${reportsDir}")).
        listFiles({ it.name.matches(".*[.]xml") } as FileFilter)

    def warnings = new HashMap<String, List<String>>()
    xmlReports.each {  reportFile ->
      def root = slurper.parse(reportFile)
      def warningList = new LinkedList<String>();
      warnings.put(reportFile, warningList)

      root."**".findAll({ it.name() == "failure"}).each { f ->
        warningList.add("${f.'@message'}:${f.text()}")
      }
    }

    def message = new StringBuffer()
    message.append("\nLINT AND TEST WARNINGS\n")
    message.append("------------------------\n")
    warnings.each { k, v ->
      if (v.size() > 0) {
        message.append("$k\n")
        v.each { w ->
          message.append("\t$w".replace("\n", "\n\t\t"))
          message.append("\n")
        }
      }
    }

    message.append("\nSUMMARY\n")
    def count = 0;
    warnings.each { k, v ->
      def fname = String.format("%30.30s", k.name)
      count += v.size()
      message.append("${fname} : ${v.size()} warnings\n")
    }

    if (count && !project.hasProperty("noFailForSourceProblems")) {
      System.err.println(message)
      throw new GradleException("Build Failed with ${count} source code issues!")
    }

    println(message)
  }
}


task verifyDockerUser {
  doLast {
    if (!project.hasProperty("dockerUser") || dockerUser == null || dockerUser.trim().isEmpty()) {
      throw new org.gradle.api.InvalidUserDataException("dockerUser has not been set")
    }
  }
}


project.tasks.dockerPrepare.dependsOn project.tasks.verifyDockerUser
project.tasks.dockerPrepare.dependsOn project.tasks.test
project.tasks.dockerRun.dependsOn project.tasks.docker
project.tasks.dockerPush.dependsOn project.tasks.docker
// needs to be set for dependency stuff to work, couldn't find a better way
project.tasks.dockerfileZip.archiveFile.set(new File(dockerZipFile))

docker {
  description 'builds the docker image, tagging with the  dockerUser and version properties'

  copySpec.from("${buildDir}").into(".").include(
      "setup.py",
      setupBuildProperties.outputs.files.singleFile.name,
      generateReqs.outputs.files.singleFile.name,
      "src/main/py/${rootPackage}/**")

  name dockerUser + "/${project.name}:" + version
  dockerfile file('Dockerfile')

}

dockerPush {}

dockerRun {
  description 'start the docker container, using property dockerLocalPort as a port'
  group 'docker'
  name 'nlp-insights'
  image dockerUser + '/nlp-insights:' + version
  ports "${dockerLocalPort}:5000"
}


// make all tasks run after clean (except clean itself)
tasks.each {
  if (it != clean) {
    it.mustRunAfter clean
  }
}