Ranger REST API SmokeTests with ROBOT Framework

.github/workflows/maven.yml

@@ -42,7 +42,7 @@ jobs:
           distribution: 'temurin'
           cache: maven
       - name: build (8)
-        run: mvn -T 8 clean install --no-transfer-progress -B -V
+        run: mvn -T 8 clean install -DskipTests --no-transfer-progress -B -V
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -66,11 +66,10 @@ jobs:
         with:
           name: target-11
           path: target/*
-  
+
   docker-build:
     needs:
       - build-8
-      - build-11
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -134,7 +133,8 @@ jobs:
           -f docker-compose.ranger-hive.yml \
           -f docker-compose.ranger-knox.yml \
           -f docker-compose.ranger-ozone.yml up -d
-      - name: Check status of containers and remove them
+
+      - name: Check status of containers
         run: | 
           sleep 60
           containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
@@ -150,8 +150,35 @@ jobs:
           
           if [[ $flag == true ]]; then
               echo "All required containers are up and running";
-              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
-          else
-              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
-              exit 1;
           fi
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      # Install Robot Framework and dependencies
+      - name: Install Robot Framework
+        run: |
+          python -m pip install --upgrade pip
+          python --version
+          pip install robotframework
+          pip install robotframework-requests
+          pip install robotframework-jsonlibrary
+          robot --version || true
+
+      - name: Run Ranger REST API SmokeTests
+        run: |
+          cd dev-support/smoketests/ranger
+          mkdir -p /tmp/smoketests/ranger
+          robot --outputdir /tmp/smoketests/ranger --loglevel DEBUG -P apitests policy_management.robot user_management.robot
+
+      - name: Upload Robot Framework Test Result Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: Robot-Framework-Artifacts
+          path: /tmp/smoketests/ranger
+
+      - name: Remove Containers
+        run: |
+          docker stop $(docker ps -q) && docker rm $(docker ps -aq);

dev-support/smoketests/ranger/apitests/__init__.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

dev-support/smoketests/ranger/apitests/policy_management.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from apache_ranger.model.ranger_service import *
+from apache_ranger.client.ranger_client import *
+from apache_ranger.model.ranger_policy import *
+
+
+class TestPolicyManagement:
+    ROBOT_LIBRARY_SCOPE = 'SUITE'
+
+    def __init__(self, ranger_url, username, password):
+        self.ranger = RangerClient(ranger_url, (username, password))
+        self.ranger.session.verify = False
+        return
+
+    def get_hive_policy(self, service_name, policy_name):
+        return self.ranger.get_policy(service_name, policy_name)
+
+    def delete_hive_policy(self, service_name, policy_name):
+        return self.ranger.delete_policy(service_name, policy_name)
+
+    def create_hive_policy(self, service_name, policy_name, db_name):
+        policy = RangerPolicy()
+        policy.service = service_name
+        policy.name = policy_name
+        policy.resources = {'database': RangerPolicyResource({'values': [db_name]}),
+                            'table': RangerPolicyResource({'values': ['test_tbl']}),
+                            'column': RangerPolicyResource({'values': ['*']})}
+
+        allowItem1 = RangerPolicyItem()
+        allowItem1.users = ['admin']
+        allowItem1.accesses = [RangerPolicyItemAccess({'type': 'create'}),
+                               RangerPolicyItemAccess({'type': 'alter'})]
+
+        denyItem1 = RangerPolicyItem()
+        denyItem1.users = ['admin']
+        denyItem1.accesses = [RangerPolicyItemAccess({'type': 'drop'})]
+
+        policy.policyItems = [allowItem1]
+        policy.denyPolicyItems = [denyItem1]
+
+        print(f'Creating policy: name={policy.name}')
+
+        created_policy = self.ranger.create_policy(policy)
+
+        print(f'Created policy: name={created_policy.name}, id={created_policy.id}')
+        return created_policy
+
+    def get_all_policies(self):
+        all_policies = self.ranger.find_policies()
+        return all_policies
+
+
+class TestServiceManagement:
+    ROBOT_LIBRARY_SCOPE = 'SUITE'
+
+    def __init__(self, ranger_url, username, password):
+        self.ranger = RangerClient(ranger_url, (username, password))
+        self.ranger.session.verify = False
+        return
+
+    def create_service(self, service_name, service_type, configs):
+        service = RangerService()
+        service.name = service_name
+        service.type = service_type
+        service.configs = configs
+        return self.ranger.create_service(service)
+
+    def delete_service(self, service_name):
+        return self.ranger.delete_service(service_name)
+

dev-support/smoketests/ranger/apitests/user_management.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from apache_ranger.client.ranger_client import *
+from apache_ranger.utils import *
+from apache_ranger.model.ranger_user_mgmt import *
+from apache_ranger.client.ranger_user_mgmt_client import *
+
+
+class TestUserManagement:
+    def __init__(self, ranger_url, username, password):
+        self.ranger = RangerClient(ranger_url, (username, password))
+        self.ranger.session.verify = False
+        self.ugclient = RangerUserMgmtClient(self.ranger)
+        return
+
+    ROBOT_LIBRARY_SCOPE = 'SUITE'
+
+    def find_users(self):
+        print('Listing all users!')
+        users = self.ugclient.find_users()
+        print(f'{len(users.list)} users found')
+        return users
+
+    def find_groups(self):
+        print('Listing all groups!')
+        groups = self.ugclient.find_groups()
+        print(f'{len(groups.list)} groups found')
+        return groups
+
+    def create_user(self, user_name):
+        user = RangerUser({'name': user_name,
+                           'firstName': user_name,
+                           'lastName': 'lnu',
+                           'emailAddress': user_name + '@test.org',
+                           'password': 'Welcome1',
+                           'userRoleList': ['ROLE_USER'],
+                           'otherAttributes': '{ "dept": "test" }'})
+
+        created_user = self.ugclient.create_user(user)
+        print(f'User {created_user.name} created!')
+        return created_user
+
+    def create_group(self, group_name):
+        group = RangerGroup({'name': group_name, 'otherAttributes': '{ "dept": "test" }'})
+        created_group = self.ugclient.create_group(group)
+        print(f'Group {created_group.name} created!')
+        return created_group
+
+    def add_to_group(self, group_name, group_id, user_id):
+        group_user = RangerGroupUser({'name': group_name, 'parentGroupId': group_id, 'userId': user_id})
+        created_group_user = self.ugclient.create_group_user(group_user)
+        print(f'Created group-user: {created_group_user}')
+        return created_group_user
+
+    def list_users_in_group(self, group_name):
+        users = self.ugclient.get_users_in_group(group_name)
+        return users
+
+    def list_groups_for_user(self, user_name):
+        groups = self.ugclient.get_groups_for_user(user_name)
+        return groups
+
+    def list_group_users(self):
+        group_users = self.ugclient.find_group_users()
+        print(f'{len(group_users.list)} group-users found')
+
+        for group_user in group_users.list:
+            print(f'id: {group_user.id}, groupId: {group_user.parentGroupId}, userId: {group_user.userId}')
+        return group_users
+
+    def delete_user_by_id(self, id):
+        self.ugclient.delete_user_by_id(id, True)
+        return
+
+    def delete_group_by_id(self, id):
+        self.ugclient.delete_group_by_id(id, True)
+        return
+
+    def delete_group_user_by_id(self, id):
+        self.ugclient.delete_group_user_by_id(id)
+        return
+

dev-support/smoketests/ranger/policy_management.robot

@@ -0,0 +1,89 @@
+*** Settings ***
+Library        policy_management.TestPolicyManagement               http://localhost:6080    admin    rangerR0cks!
+Library        policy_management.TestServiceManagement              http://localhost:6080    admin    rangerR0cks!
+Library        Collections
+Library        JSONLibrary
+
+*** Variables ***
+${HIVE_DEFAULT_SVC}        dev_hive
+${HIVE_DEFAULT_POLICY}     all - global
+${HIVE_TEST_SVC}           test_hive
+${HIVE_TEST_POLICY}        test_policy
+@{TEST_DB}                 test_db
+
+*** Test Cases ***
+Create Hive Test Service
+    [Documentation]    Create a test Hive service with specific configurations.
+    ${configs}         Create Dictionary    username=hive    password=hive    jdbc.driverClassName=org.apache.hive.jdbc.HiveDriver    jdbc.url=jdbc:hive2://ranger-hadoop:10000    hadoop.security.authorization=${true}
+    ${response}        Create Service       test_hive        hive             ${configs}
+    Log                ${response}
+
+
+Delete Hive Test Service
+    [Documentation]    Delete the test Hive service.
+    ${response}        Delete Service    ${HIVE_TEST_SVC}
+
+
+Get All Policies
+    [Documentation]    Get all policies in Ranger.
+    ${response}        Get All Policies
+    ${service}         Get Value From Json    ${response}    $[0].service
+    Should Be Equal    ${service}[0]          dev_hdfs
+
+
+Validate Response - Get Default Hive Policy
+    [Documentation]    Validate Response from the default hive policy: all - global
+    ${response}        Get Hive Policy        ${HIVE_DEFAULT_SVC}   ${HIVE_DEFAULT_POLICY}
+
+    ${service}         Get Value From Json    ${response}    $.service
+    ${name}            Get Value From Json    ${response}    $.name
+    ${isEnabled}       Get Value From Json    ${response}    $.isEnabled
+    ${users}           Get Value From Json    ${response}    $.policyItems[0].users
+
+    Should Be Equal              ${service}[0]    ${HIVE_DEFAULT_SVC}
+    Should Be Equal              ${name}[0]       ${HIVE_DEFAULT_POLICY}
+    Should Be True               ${isEnabled}
+    List Should Contain Value    ${users}[0]      hive
+    Log    ${response}
+
+
+Create Hive Test Policy
+    [Documentation]         Create a test policy in Default Hive Service
+    ${response}             Create Hive Policy    dev_hive    test_policy_78     test_db
+
+    ${id}                   Get Value From Json    ${response}    $.id
+    Set Suite Variable      ${POLICY_ID}    ${id}
+    Log                     ${response}
+
+
+Delete Hive Test Policy
+    ${response}     Delete Hive Policy       dev_hive    test_policy_78
+    Log             ${response}
+
+# Depends on an earlier Test
+Validate Successive Hive Test Policy
+    [Documentation]     Create the test policy again in Default Hive Service
+    ${response}         Create Hive Policy    dev_hive    test_policy_78     test_db
+
+    ${id}               Get Value From Json    ${response}    $.id
+    ${result}           Evaluate    ${POLICY_ID}[0] + 1
+    Should Be Equal     ${id}[0]    ${result}
+    Log                 ${response}
+
+
+Delete Successive Hive Test Policy
+    ${response}     Delete Hive Policy          dev_hive    test_policy_78
+    Log             ${response}
+
+
+Create 100 Policies
+    FOR    ${i}    IN RANGE    1    101
+        ${policy_name}      Create Hive Policy    dev_hive    policy_${i}     test_db_${i}
+        Log                 Created policy: ${policy_name}
+    END
+
+
+Delete 100 Policies
+    FOR    ${i}    IN RANGE    1    101
+        Delete Hive Policy      ${HIVE_DEFAULT_SVC}    policy_${i}
+    END