[HWORKS-697] Add airflow user to project (logicalclocks#1520)

smkniazi · Oct 3, 2023 · aa3c675 · aa3c675
1 parent 0246744
commit aa3c675
Show file tree

Hide file tree

Showing 17 changed files with 63 additions and 755 deletions.
diff --git a/hopsworks-IT/src/test/ruby/spec/airflow_spec.rb b/hopsworks-IT/src/test/ruby/spec/airflow_spec.rb
@@ -45,6 +45,12 @@
         with_valid_project
       end
 
+      it 'should add airflow user to project' do
+        get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/projectMembers"
+        airflow_member = json_body.detect { |e| e[:user][:email] == "[email protected]" }
+        expect(airflow_member).not_to be_nil
+      end
+
       it "should be able to compose DAG" do
         get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/airflow/secretDir"
         expect_status_details(200)

diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/airflow/AirflowService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/airflow/AirflowService.java
@@ -23,7 +23,6 @@
 import io.hops.hopsworks.audit.logger.annotation.Logged;
 import io.hops.hopsworks.common.airflow.AirflowController;
 import io.hops.hopsworks.common.airflow.AirflowDagDTO;
-import io.hops.hopsworks.common.airflow.AirflowJWTManager;
 import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.exceptions.AirflowException;
 import io.hops.hopsworks.jwt.annotation.JWTRequired;
@@ -55,8 +54,6 @@ public class AirflowService {
   @EJB
   private JWTHelper jwtHelper;
   @EJB
-  private AirflowJWTManager airflowJWTManager;
-  @EJB
   private AirflowController airflowController;
   private Integer projectId;
   // No @EJB annotation for Project, it's injected explicitly in ProjectService.
@@ -80,20 +77,6 @@ public Integer getProjectId() {
     return projectId;
   }
 
-  @POST
-  @Path("/jwt")
-  @Produces(MediaType.APPLICATION_JSON)
-  @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST})
-  @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"})
-  @ApiOperation(value = "Generate a JWT for Airflow usage and store it in project's secret directory in Airflow")
-  public Response storeAirflowJWT(@Context HttpServletRequest req,
-                                  @Context SecurityContext sc) throws AirflowException {
-    Users user = jwtHelper.getUserPrincipal(sc);
-    airflowJWTManager.prepareSecurityMaterial(user, project, JWT_AUDIENCE);
-    return Response.noContent().build();
-  }
-
-
   @POST
   @Path("/dag")
   @Produces(MediaType.APPLICATION_JSON)

diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsResource.java
@@ -118,7 +118,7 @@ public JobsResource setProject(Integer projectId) {
   @Produces(MediaType.APPLICATION_JSON)
   @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER})
   @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB},
-    allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
+    allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER", "AGENT"})
   @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
   public Response getAll(
       @BeanParam Pagination pagination,
@@ -142,7 +142,7 @@ public Response getAll(
   @Produces(MediaType.APPLICATION_JSON)
   @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST})
   @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB},
-    allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
+    allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER", "AGENT"})
   @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
   public Response getJob(@PathParam("name") String name,
                          @BeanParam JobsBeanParam jobsBeanParam,

diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/executions/ExecutionsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/executions/ExecutionsResource.java
@@ -27,6 +27,7 @@
 import io.hops.hopsworks.common.api.ResourceRequest;
 import io.hops.hopsworks.common.jobs.JobLogDTO;
 import io.hops.hopsworks.common.jobs.execution.ExecutionController;
+import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.GenericException;
 import io.hops.hopsworks.exceptions.JobException;
 import io.hops.hopsworks.exceptions.ProjectException;
@@ -70,6 +71,8 @@ public class ExecutionsResource {
   private ExecutionController executionController;
   @EJB
   private ExecutionsBuilder executionsBuilder;
+  @EJB
+  private Settings settings;
 
 
   @EJB
@@ -89,7 +92,8 @@ public ExecutionsResource setJob(Jobs job) {
   @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER})
   @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB},
     allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
-  @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
+  @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER",
+      "AGENT"})
   public Response getExecutions(
     @BeanParam Pagination pagination,
     @BeanParam ExecutionsBeanParam executionsBeanParam,
@@ -115,7 +119,8 @@ public Response getExecutions(
   @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER})
   @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB},
     allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
-  @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
+  @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER",
+      "AGENT"})
   public Response getExecution(@ApiParam(value = "execution id", required = true) @PathParam("id") Integer id,
                                @BeanParam ExecutionsBeanParam executionsBeanParam,
                                @Context UriInfo uriInfo,
@@ -160,7 +165,8 @@ public Response stopExecution(
   @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER})
   @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB},
     allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
-  @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
+  @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER",
+      "AGENT"})
   public Response startExecution(
     @ApiParam(value = "Arguments for executing the job") String args,
     @Context HttpServletRequest req,
@@ -169,6 +175,11 @@ public Response startExecution(
 
     Users user = jWTHelper.getUserPrincipal(sc);
 
+    // run job as job owner if user is airflow
+    if (user.getUsername().equals(settings.getAirflowUser())) {
+      user = job.getCreator();
+    }
+
     Execution exec;
     if(!Strings.isNullOrEmpty(job.getJobConfig().getDefaultArgs()) && Strings.isNullOrEmpty(args)) {
       exec = executionController.start(job, job.getJobConfig().getDefaultArgs(), user);

diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java
@@ -277,7 +277,7 @@ public Response getAllProjects(@Context HttpServletRequest req, @Context Securit
   @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB},
     allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
   @ApiKeyRequired(acceptedScopes = {ApiScope.PROJECT},
-    allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"})
+    allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER", "AGENT"})
   @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER})
   @Produces(MediaType.APPLICATION_JSON)
   public Response getProjectByName(@PathParam("projectName") String projectName,
@@ -290,7 +290,7 @@ public Response getProjectByName(@PathParam("projectName") String projectName,
   @GET
   @Path("/asShared/getProjectInfo/{projectName}")
   @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"})
-  @ApiKeyRequired( acceptedScopes = {ApiScope.PROJECT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"})
+  @ApiKeyRequired( acceptedScopes = {ApiScope.PROJECT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "AGENT"})
   @Produces(MediaType.APPLICATION_JSON)
   public Response getProjectByNameAsShared(@PathParam("projectName") String projectName,
     @Context HttpServletRequest req, @Context SecurityContext sc) throws ProjectException, GenericException {