Skip to content

Commit

Permalink
If no mapred.mesos.executor.uri is set, rely on mapred.mesos.executor…
Browse files Browse the repository at this point in the history
….directory, if set

Retrieving the URI for the hadoop distribution is not always necessary for
every task submitted to Mesos, in clusters with shared storage, the
distribution can be shared.
  • Loading branch information
hansbogert committed Mar 4, 2015
1 parent 0cfb366 commit fdbb4f2
Showing 1 changed file with 15 additions and 7 deletions.
22 changes: 15 additions & 7 deletions src/main/java/org/apache/hadoop/mapred/ResourcePolicy.java
Original file line number Diff line number Diff line change
Expand Up @@ -383,16 +383,22 @@ public void resourceOffers(SchedulerDriver schedulerDriver,
}

String uri = scheduler.conf.get("mapred.mesos.executor.uri");
if (uri == null) {
String directory = scheduler.conf.get("mapred.mesos.executor.directory");
boolean isUriSet = uri != null && !uri.equals("");
boolean isDirectorySet = directory != null && !directory.equals("");

if (!isUriSet && !isDirectorySet) {
throw new RuntimeException(
"Expecting configuration property 'mapred.mesos.executor'");
}

String directory = scheduler.conf.get("mapred.mesos.executor.directory");
if (directory == null || directory.equals("")) {
} else if (isUriSet && isDirectorySet) {
throw new RuntimeException(
"Conflicting properties 'mapred.mesos.executor.uri' and 'mapred.mesos.executor.directory', only one can be set");
} else if (!isDirectorySet) {
LOG.info("URI: " + uri + ", name: " + new File(uri).getName());

directory = new File(uri).getName().split("\\.")[0] + "*";
} else if (!isUriSet) {
LOG.info("mapred.mesos.executor.uri is not set, relying on configured 'mapred.mesos.executor.directory' for working Hadoop distribution");
}

String command = scheduler.conf.get("mapred.mesos.executor.command");
Expand All @@ -403,8 +409,10 @@ public void resourceOffers(SchedulerDriver schedulerDriver,
CommandInfo.Builder commandInfo = CommandInfo.newBuilder();
commandInfo
.setEnvironment(envBuilder)
.setValue(String.format("cd %s && %s", directory, command))
.addUris(CommandInfo.URI.newBuilder().setValue(uri));
.setValue(String.format("cd %s && %s", directory, command));
if (uri != null) {
commandInfo.addUris(CommandInfo.URI.newBuilder().setValue(uri));
}

// Populate ContainerInfo if needed
String containerImage = scheduler.conf.get("mapred.mesos.container.image");
Expand Down

0 comments on commit fdbb4f2

Please sign in to comment.