Skip to content

Commit

Permalink
move add default ACL of object to OM
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChenSammi
ChenSammi committed Nov 22, 2024
1 parent de55d6f commit 48f8618
Show file tree
Hide file tree
Showing 15 changed files with 372 additions and 329 deletions.
74 changes: 12 additions & 62 deletions hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,6 @@
import org.apache.hadoop.ozone.om.helpers.OmTenantArgs;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
Expand All @@ -141,8 +140,6 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import org.apache.hadoop.ozone.security.GDPRSymmetricKey;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
import org.apache.hadoop.ozone.security.acl.OzoneAclConfig;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
Expand Down Expand Up @@ -175,7 +172,6 @@
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY;
Expand All @@ -184,8 +180,6 @@
import static org.apache.hadoop.ozone.OzoneConsts.MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD;
import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MAXIMUM_ACCESS_ID_LENGTH;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE;

/**
* Ozone RPC Client Implementation, it connects to OM, SCM and DataNode
Expand All @@ -209,8 +203,6 @@ public class RpcClient implements ClientProtocol {
private final XceiverClientFactory xceiverClientManager;
private final UserGroupInformation ugi;
private UserGroupInformation s3gUgi;
private final ACLType[] userRights;
private final ACLType[] groupRights;
private final ClientId clientId = ClientId.randomId();
private final boolean unsafeByteBufferConversion;
private Text dtService;
Expand Down Expand Up @@ -247,8 +239,6 @@ public RpcClient(ConfigurationSource conf, String omServiceId)
OzoneAclConfig aclConfig = this.conf.getObject(OzoneAclConfig.class);
replicationConfigValidator =
this.conf.getObject(ReplicationConfigValidator.class);
this.userRights = aclConfig.getUserDefaultRights();
this.groupRights = aclConfig.getGroupDefaultRights();

this.clientConfig = conf.getObject(OzoneClientConfig.class);
this.ecReconstructExecutor = MemoizedSupplier.valueOf(() -> createThreadPoolExecutor(
Expand Down Expand Up @@ -449,13 +439,6 @@ public void createVolume(String volumeName, VolumeArgs volArgs)
ugi.getShortUserName() : volArgs.getOwner();
long quotaInNamespace = volArgs.getQuotaInNamespace();
long quotaInBytes = volArgs.getQuotaInBytes();
List<OzoneAcl> listOfAcls =
OzoneAclUtil.getAclList(UserGroupInformation.createRemoteUser(owner), userRights, groupRights);
//ACLs from VolumeArgs
List<OzoneAcl> volumeAcls = volArgs.getAcls();
if (volumeAcls != null) {
listOfAcls.addAll(volumeAcls);
}

OmVolumeArgs.Builder builder = OmVolumeArgs.newBuilder();
builder.setVolume(volumeName);
Expand All @@ -465,11 +448,14 @@ public void createVolume(String volumeName, VolumeArgs volArgs)
builder.setQuotaInNamespace(quotaInNamespace);
builder.setUsedNamespace(0L);
builder.addAllMetadata(volArgs.getMetadata());

//Remove duplicates and add ACLs
for (OzoneAcl ozoneAcl :
listOfAcls.stream().distinct().collect(Collectors.toList())) {
builder.addOzoneAcls(ozoneAcl);
//ACLs from VolumeArgs
List<OzoneAcl> volumeAcls = volArgs.getAcls();
if (volumeAcls != null) {
//Remove duplicates and add ACLs
for (OzoneAcl ozoneAcl :
volumeAcls.stream().distinct().collect(Collectors.toList())) {
builder.addOzoneAcls(ozoneAcl);
}
}

if (volArgs.getQuotaInBytes() == 0) {
Expand Down Expand Up @@ -659,17 +645,6 @@ public void createBucket(
.setKeyName(bucketArgs.getEncryptionKey()).build();
}

List<OzoneAcl> listOfAcls = getAclList();
//ACLs from BucketArgs
if (bucketArgs.getAcls() != null) {
listOfAcls.addAll(bucketArgs.getAcls());
}
// Link bucket default acl
if (bucketArgs.getSourceVolume() != null
&& bucketArgs.getSourceBucket() != null) {
listOfAcls.add(linkBucketDefaultAcl());
}

OmBucketInfo.Builder builder = OmBucketInfo.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
Expand All @@ -680,10 +655,13 @@ public void createBucket(
.setSourceBucket(bucketArgs.getSourceBucket())
.setQuotaInBytes(bucketArgs.getQuotaInBytes())
.setQuotaInNamespace(bucketArgs.getQuotaInNamespace())
.setAcls(listOfAcls.stream().distinct().collect(Collectors.toList()))
.setBucketLayout(bucketLayout)
.setOwner(owner);

if (bucketArgs.getAcls() != null) {
builder.setAcls(bucketArgs.getAcls());
}

if (bek != null) {
builder.setBucketEncryptionKey(bek);
}
Expand Down Expand Up @@ -744,16 +722,6 @@ private static void verifySpaceQuota(long quota) throws OMException {
}
}

/**
* Helper function to get default acl list for current user.
*
* @return listOfAcls
* */
private List<OzoneAcl> getAclList() {
UserGroupInformation realUserInfo = getRealUserInfo();
return OzoneAclUtil.getAclList(realUserInfo, userRights, groupRights);
}

/**
* Helper function to get the actual operating user.
*
Expand All @@ -769,16 +737,6 @@ private UserGroupInformation getRealUserInfo() {
return ugi;
}

/**
* Link bucket default acl defined [world::rw]
* which is similar to Linux POSIX symbolic.
*
* @return OzoneAcl
*/
private OzoneAcl linkBucketDefaultAcl() {
return new OzoneAcl(ACLIdentityType.WORLD, "", ACCESS, READ, WRITE);
}

/**
* Get a valid Delegation Token.
*
Expand Down Expand Up @@ -1418,7 +1376,6 @@ public OzoneOutputStream createKey(
.setReplicationConfig(replicationConfig)
.addAllMetadataGdpr(metadata)
.addAllTags(tags)
.setAcls(getAclList())
.setLatestVersionLocation(getLatestVersionLocation)
.setOwnerName(ownerName);

Expand Down Expand Up @@ -1527,7 +1484,6 @@ public OzoneDataStreamOutput createStreamKey(
.addAllMetadataGdpr(metadata)
.addAllTags(tags)
.setSortDatanodesInPipeline(true)
.setAcls(getAclList())
.setOwnerName(ownerName);

OpenKeySession openKey = ozoneManagerClient.openKey(builder.build());
Expand Down Expand Up @@ -1946,7 +1902,6 @@ public OmMultipartInfo initiateMultipartUpload(String volumeName,
.setBucketName(bucketName)
.setKeyName(keyName)
.setReplicationConfig(replicationConfig)
.setAcls(getAclList())
.addAllMetadataGdpr(metadata)
.setOwnerName(ownerName)
.addAllTags(tags)
Expand Down Expand Up @@ -1983,7 +1938,6 @@ private OpenKeySession newMultipartOpenKey(
.setMultipartUploadID(uploadID)
.setMultipartUploadPartNumber(partNumber)
.setSortDatanodesInPipeline(sortDatanodesInPipeline)
.setAcls(getAclList())
.setOwnerName(ownerName)
.build();
return ozoneManagerClient.openKey(keyArgs);
Expand Down Expand Up @@ -2055,7 +2009,6 @@ public OmMultipartUploadCompleteInfo completeMultipartUpload(
.setBucketName(bucketName)
.setKeyName(keyName)
.setMultipartUploadID(uploadID)
.setAcls(getAclList())
.setOwnerName(ownerName)
.build();

Expand Down Expand Up @@ -2160,7 +2113,6 @@ public void createDirectory(String volumeName, String bucketName,
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
.setAcls(getAclList())
.setOwnerName(ownerName)
.build();
ozoneManagerClient.createDirectory(keyArgs);
Expand Down Expand Up @@ -2241,7 +2193,6 @@ public OzoneOutputStream createFile(String volumeName, String bucketName,
.setKeyName(keyName)
.setDataSize(size)
.setReplicationConfig(replicationConfig)
.setAcls(getAclList())
.setLatestVersionLocation(getLatestVersionLocation)
.setOwnerName(ownerName)
.build();
Expand Down Expand Up @@ -2273,7 +2224,6 @@ public OzoneDataStreamOutput createStreamFile(String volumeName,
.setKeyName(keyName)
.setDataSize(size)
.setReplicationConfig(replicationConfig)
.setAcls(getAclList())
.setLatestVersionLocation(getLatestVersionLocation)
.setSortDatanodesInPipeline(true)
.setOwnerName(ownerName)
Expand Down
27 changes: 27 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@

package org.apache.hadoop.ozone.om.helpers;

import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
import org.apache.hadoop.ozone.security.acl.OzoneAclConfig;
import org.apache.hadoop.ozone.security.acl.RequestContext;

import java.io.IOException;
Expand All @@ -38,6 +40,8 @@
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.GROUP;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE;

/**
* Helper class for ozone acls operations.
Expand All @@ -47,6 +51,29 @@ public final class OzoneAclUtil {
private OzoneAclUtil() {
}

static ACLType[] userRights;
static ACLType[] groupRights;

public static List<OzoneAcl> getDefaultAclList(UserGroupInformation ugi, OzoneConfiguration conf) {
// Get default acl rights for user and group.
if (userRights == null || groupRights == null) {
OzoneAclConfig aclConfig = conf.getObject(OzoneAclConfig.class);
userRights = aclConfig.getUserDefaultRights();
groupRights = aclConfig.getGroupDefaultRights();
}
return getAclList(ugi, userRights, groupRights);
}

/**
* Link bucket default acl defined [world::rw]
* which is similar to Linux POSIX symbolic.
*
* @return OzoneAclInfo
*/
public static OzoneAcl linkBucketDefaultAcl() {
return new OzoneAcl(IAccessAuthorizer.ACLIdentityType.WORLD, "", ACCESS, READ, WRITE);
}

/**
* Helper function to get access acl list for current user.
*
Expand Down
44 changes: 40 additions & 4 deletions ...ne/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,21 @@ public void testKeyDefaultACL() throws Exception {
try (OzoneClient client = cluster.newClient()) {
ObjectStore objectStore = client.getObjectStore();
objectStore.createVolume(volumeName);
setVolumeAcl(objectStore, volumeName, "world::a");
addVolumeAcl(objectStore, volumeName, "world::a");

// verify volume ACLs. This volume will have 2 default ACLs, plus above one added
OzoneObj obj = OzoneObjInfo.Builder.newBuilder().setVolumeName(volumeName)
.setResType(OzoneObj.ResourceType.VOLUME)
.setStoreType(OZONE).build();
List<OzoneAcl> acls = objectStore.getAcl(obj);
assertEquals(3, acls.size());
assertEquals(adminUser.getShortUserName(), acls.get(0).getName());
OzoneAclConfig aclConfig = cluster.getConf().getObject(OzoneAclConfig.class);
assertArrayEquals(aclConfig.getUserDefaultRights(), acls.get(0).getAclList().toArray());
assertEquals(adminUser.getPrimaryGroupName(), acls.get(1).getName());
assertArrayEquals(aclConfig.getGroupDefaultRights(), acls.get(1).getAclList().toArray());
assertEquals("WORLD", acls.get(2).getName());
assertArrayEquals(aclConfig.getUserDefaultRights(), acls.get(2).getAclList().toArray());
}

// set LoginUser as user3
Expand All @@ -238,18 +252,30 @@ public void testKeyDefaultACL() throws Exception {
volume.createBucket(bucketName, omBucketArgs);
OzoneBucket ozoneBucket = volume.getBucket(bucketName);

// verify bucket default ACLs
OzoneObj obj = OzoneObjInfo.Builder.newBuilder().setVolumeName(volume.getName())
.setBucketName(ozoneBucket.getName()).setResType(OzoneObj.ResourceType.BUCKET)
.setStoreType(OZONE).build();
List<OzoneAcl> acls = objectStore.getAcl(obj);
assertEquals(2, acls.size());
assertEquals(user3.getShortUserName(), acls.get(0).getName());
OzoneAclConfig aclConfig = cluster.getConf().getObject(OzoneAclConfig.class);
assertArrayEquals(aclConfig.getUserDefaultRights(), acls.get(0).getAclList().toArray());
assertEquals(user3.getPrimaryGroupName(), acls.get(1).getName());
assertArrayEquals(aclConfig.getGroupDefaultRights(), acls.get(1).getAclList().toArray());

// verify key default ACLs
int length = 10;
byte[] input = new byte[length];
Arrays.fill(input, (byte) 96);
String keyName = UUID.randomUUID().toString();
createKey(ozoneBucket, keyName, length, input);
OzoneObj obj = OzoneObjInfo.Builder.newBuilder().setVolumeName(volume.getName())
obj = OzoneObjInfo.Builder.newBuilder().setVolumeName(volume.getName())
.setBucketName(ozoneBucket.getName()).setKeyName(keyName)
.setResType(OzoneObj.ResourceType.KEY).setStoreType(OZONE).build();
List<OzoneAcl> acls = objectStore.getAcl(obj);
acls = objectStore.getAcl(obj);
assertEquals(2, acls.size());
assertEquals(user3.getShortUserName(), acls.get(0).getName());
OzoneAclConfig aclConfig = cluster.getConf().getObject(OzoneAclConfig.class);
assertArrayEquals(aclConfig.getUserDefaultRights(), acls.get(0).getAclList().toArray());
assertEquals(user3.getPrimaryGroupName(), acls.get(1).getName());
assertArrayEquals(aclConfig.getGroupDefaultRights(), acls.get(1).getAclList().toArray());
Expand Down Expand Up @@ -314,6 +340,16 @@ private void setVolumeAcl(ObjectStore objectStore, String volumeName,
assertTrue(objectStore.setAcl(obj, OzoneAcl.parseAcls(aclString)));
}

/**
* Helper function to add volume ACL.
*/
private void addVolumeAcl(ObjectStore objectStore, String volumeName,
String aclString) throws IOException {
OzoneObj obj = OzoneObjInfo.Builder.newBuilder().setVolumeName(volumeName)
.setResType(OzoneObj.ResourceType.VOLUME).setStoreType(OZONE).build();
assertTrue(objectStore.addAcl(obj, OzoneAcl.parseAcl(aclString)));
}

/**
* Helper function to set bucket ACL.
*/
Expand Down
Loading

0 comments on commit 48f8618

Please sign in to comment.