From 8915003e25b363f072aa3bea3d09335b5aba434b Mon Sep 17 00:00:00 2001 From: Kumar Krishna Date: Fri, 22 Jul 2016 13:31:24 -0700 Subject: [PATCH 01/15] multi compose file support --- pom.xml | 14 +- .../paypal/mesos/executor/CommandBuilder.java | 49 +++-- .../mesos/executor/ComposeFileListModule.java | 18 ++ .../mesos/executor/DockerComposeExecutor.java | 169 +++++++++++++++--- .../mesos/executor/ExecutorComponent.java | 2 +- .../paypal/mesos/executor/ExecutorModule.java | 21 ++- .../mesos/executor/FileFetcherModule.java | 32 ---- .../executor/compose/ComposeFileList.java | 12 ++ .../executor/compose/ComposeFileListImpl.java | 134 ++++++++++++++ .../ComposeRewriteHelper.java} | 20 ++- .../fetcher/DockerComposeFileFetcher.java | 104 ----------- .../mesos/executor/fetcher/FileFetcher.java | 13 -- .../executor/monitoring/ComposeMonitor.java | 16 +- .../pluginapi/ComposeExecutorPlugin.java | 21 +++ .../pluginapi/ComposeExecutorPluginImpl.java | 51 ++++++ .../mesos/executor/utils/ProcessUtils.java | 3 + 16 files changed, 485 insertions(+), 194 deletions(-) create mode 100644 src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java delete mode 100644 src/main/java/com/paypal/mesos/executor/FileFetcherModule.java create mode 100644 src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java create mode 100644 src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java rename src/main/java/com/paypal/mesos/executor/{fetcher/DockerRewriteHelper.java => compose/ComposeRewriteHelper.java} (84%) delete mode 100644 src/main/java/com/paypal/mesos/executor/fetcher/DockerComposeFileFetcher.java delete mode 100644 src/main/java/com/paypal/mesos/executor/fetcher/FileFetcher.java create mode 100644 src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java create mode 100644 src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPluginImpl.java diff --git a/pom.xml b/pom.xml index bae5f54..9b2417d 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,11 @@ commons-exec 1.3 - + + ro.fortsoft.pf4j + pf4j + 0.13.1 + @@ -123,6 +127,14 @@ + + org.apache.maven.plugins + maven-deploy-plugin + 3.1 + + true + + diff --git a/src/main/java/com/paypal/mesos/executor/CommandBuilder.java b/src/main/java/com/paypal/mesos/executor/CommandBuilder.java index 118a7ef..ecad431 100644 --- a/src/main/java/com/paypal/mesos/executor/CommandBuilder.java +++ b/src/main/java/com/paypal/mesos/executor/CommandBuilder.java @@ -7,24 +7,53 @@ public class CommandBuilder { - public static String launchTask(String fileName){ - return "docker-compose -f "+fileName+" up"; + public static String launchTask(List files){ + StringBuffer buf = new StringBuffer("docker-compose "); + for (String file : files) { + buf.append(" -f "+file); + } + buf.append(" up"); + return buf.toString(); + //return "docker-compose -f "+fileName+" up"; } - public static String pullImages(String fileName){ - String command = "docker-compose -f "+fileName+" pull"; + public static String pullImages(List files){ + StringBuffer buf = new StringBuffer("docker-compose "); + for (String file : files) { + buf.append(" -f "+file); + } + buf.append(" pull"); + if(Config.IGNORE_PULL_FAILURES){ - command = command+" --ignore-pull-failures"; + buf.append(" --ignore-pull-failures"); } - return command; + return buf.toString(); + +// String command = "docker-compose -f "+fileName+" pull"; +// if(Config.IGNORE_PULL_FAILURES){ +// command = command+" --ignore-pull-failures"; +// } +// return command; } - public static String stopTask(String fileName){ - return "docker-compose -f "+fileName+" stop "; + public static String stopTask(List files){ + StringBuffer buf = new StringBuffer("docker-compose "); + for (String file : files) { + buf.append(" -f "+file); + } + buf.append(" stop"); + return buf.toString(); + //return "docker-compose -f "+fileName+" stop "; } - public static String getContainerIds(String fileName){ - return "docker-compose -f "+fileName + " ps -q"; + public static String getContainerIds(List files){ + StringBuffer buf = new StringBuffer("docker-compose "); + for (String file : files) { + buf.append(" -f "+file); + } + buf.append(" ps -q"); + return buf.toString(); + //return "docker-compose -f "+fileName + " ps -q"; } public static String linuxKill(List pids){ diff --git a/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java b/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java new file mode 100644 index 0000000..39e2b88 --- /dev/null +++ b/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java @@ -0,0 +1,18 @@ +package com.paypal.mesos.executor; + +import javax.inject.Singleton; + +import com.paypal.mesos.executor.compose.ComposeFileListImpl; +import com.paypal.mesos.executor.compose.ComposeFileList; + +import dagger.Module; +import dagger.Provides; + +@Module +public class ComposeFileListModule { + + @Provides @Singleton ComposeFileList provideComposeFileList(){ + return new ComposeFileListImpl(); + } + +} diff --git a/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java b/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java index c85f008..2a0f6a7 100644 --- a/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java +++ b/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java @@ -1,11 +1,14 @@ package com.paypal.mesos.executor; -import java.io.File; +import java.io.*; import java.util.ArrayList; import java.util.List; +import java.util.Map; import javax.inject.Inject; +import com.paypal.mesos.executor.compose.ComposeRewriteHelper; +import com.paypal.mesos.executor.pluginapi.ComposeExecutorPlugin; import org.apache.log4j.Logger; import org.apache.mesos.Executor; import org.apache.mesos.ExecutorDriver; @@ -17,12 +20,16 @@ import org.apache.mesos.Protos.TaskState; import org.apache.mesos.Protos.TaskStatus; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import ro.fortsoft.pf4j.DefaultPluginManager; +import ro.fortsoft.pf4j.PluginManager; import rx.Observable; import rx.Subscriber; import rx.functions.Action1; import rx.schedulers.Schedulers; -import com.paypal.mesos.executor.fetcher.FileFetcher; +import com.paypal.mesos.executor.compose.ComposeFileList; import com.paypal.mesos.executor.monitoring.ComposeMonitor; import com.paypal.mesos.executor.monitoring.ContainerDetails; import com.paypal.mesos.executor.utils.ProcessUtils; @@ -30,43 +37,69 @@ public class DockerComposeExecutor implements Executor{ private static final Logger log = Logger.getLogger(DockerComposeExecutor.class); - + private static final String GENERATED_YAML_FILE_NAME = "-generated.yml"; + - private FileFetcher fileFetcher; + private ComposeFileList fileFetcher; - private String fileName; + private List fileNames; private ExecutorDriver executorDriver; private DockerComposeProcessObserver processObserver; - - + private ExecutorInfo executorInfo; - + private ComposeMonitor podMonitor; - + + private ComposeRewriteHelper composeRewriteHelper; + + final PluginManager pluginManager = new DefaultPluginManager(); + + private boolean pluginEnabled = false; + @Inject - public DockerComposeExecutor(FileFetcher fileFetcher,DockerComposeProcessObserver processObserver,ComposeMonitor podMonitor){ + public DockerComposeExecutor(ComposeFileList fileFetcher,DockerComposeProcessObserver processObserver, + ComposeMonitor podMonitor, ComposeRewriteHelper helper){ this.fileFetcher = fileFetcher; this.processObserver = processObserver; this.podMonitor = podMonitor; + this.composeRewriteHelper = helper; + //load/start plugins needed only if plugin is loaded from separate jar +// this.pluginManager.loadPlugins(); +// this.pluginManager.startPlugins(); } @Override public void launchTask(ExecutorDriver executorDriver, TaskInfo taskInfo) { + System.out.println(" ############ Launch Task START #############"); + System.out.println(" taskInfo: "+taskInfo.toString()); + System.out.println(" executorDriver: "+executorDriver.toString()); TaskID taskId = taskInfo.getTaskId(); + + + System.out.println(" taskId: " + taskId.getValue()); + processObserver.init(this, taskId); - sendTaskStatusUpdate(executorDriver,taskId,TaskState.TASK_STARTING); + sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_STARTING); try { - File file = fileFetcher.getFile(executorInfo,taskInfo); - this.fileName = file.getAbsolutePath(); + //this.fileNames = fileFetcher.getFile(taskInfo); + this.fileNames = executePlugin(executorDriver, taskInfo); + if (this.fileNames == null || this.fileNames.size() == 0) { + sendTaskStatusUpdate(executorDriver,taskId,TaskState.TASK_FAILED); + return; + } + + this.fileNames = updateYamlFiles(this.fileNames, executorInfo, taskInfo); + //this.fileNames = file.getAbsolutePath(); + System.out.println(" fileName: "+this.fileNames); podMonitor.subscribeToChanges(new Action1() { @Override public void call(Integer exitCode) { suicide(taskId, exitCode); } }); - podMonitor.startMonitoring(fileName); + podMonitor.startMonitoring(this.fileNames); updateImagesAndStartCompose(taskId); sendTaskStatusUpdate(executorDriver,taskId,TaskState.TASK_RUNNING); }catch (Exception e) { @@ -75,25 +108,62 @@ public void call(Integer exitCode) { } } + private List executePlugin(ExecutorDriver executorDriver, TaskInfo taskInfo) { + if (this.pluginManager != null) { + List plugins = this.pluginManager.getExtensions(ComposeExecutorPlugin.class); + System.out.println(String.format("Found %d extensions for extension point '%s'", plugins.size(), ComposeExecutorPlugin.class.getName())); + if (plugins.size() > 1) { + log.error(" more than one extension implementation: "+plugins.toString()); + return null; + } + //check to ensure that only a single plugin exists.. + for (ComposeExecutorPlugin plugin : plugins) { + return plugin.launchTask(executorDriver, taskInfo); + } + }else { + log.error(" pluginManager NULL "); + } + return null; + } + + private void shutdownPlugin() { + if (this.pluginManager != null) { + List plugins = this.pluginManager.getExtensions(ComposeExecutorPlugin.class); + System.out.println(String.format("Found %d extensions for extension point '%s'", plugins.size(), ComposeExecutorPlugin.class.getName())); + for (ComposeExecutorPlugin plugin : plugins) { + plugin.shutdown(); + } + } + } + private void updateImagesAndStartCompose(TaskID taskId){ Observable.create(new Observable.OnSubscribe() { @Override public void call(Subscriber subscriber) { - String pullCommand = CommandBuilder.pullImages(fileName); + String pullCommand = CommandBuilder.pullImages(fileNames); + System.out.println(" pullCommand: "+pullCommand); int imageUpdateExitCode = ProcessUtils.executeCommand(pullCommand, null); if(imageUpdateExitCode != 0){ log.error("unable to pull updated images trying to bring the pod up with existing images"); } - String launchCommand = CommandBuilder.launchTask(fileName); + String launchCommand = CommandBuilder.launchTask(fileNames); + System.out.println(" launchCommand: "+ launchCommand); int exitCode = ProcessUtils.executeCommand(launchCommand, null); subscriber.onNext(exitCode); subscriber.onCompleted(); } }).subscribeOn(Schedulers.newThread()).observeOn(Schedulers.newThread()).subscribe(processObserver); } - + public void suicide(TaskID taskId,int exitCode){ + System.out.println(" ############## in suicide #######"); + System.out.println("taskId: "+taskId.toString() + " exitCode: "+exitCode); int stopContainers = cleanUp(); + + if (this.pluginManager != null) { + this.pluginManager.stopPlugins(); + } + if(exitCode == 0 && stopContainers == 0){ sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FINISHED); System.exit(0); @@ -110,19 +180,24 @@ public void suicide(TaskID taskId,int exitCode){ * @return killing compose and removing all images are successful */ private int cleanUp(){ - String killTask = CommandBuilder.stopTask(fileName); + System.out.println(" ############## Inside cleanUp ###########, fileName: "+fileNames); + String killTask = CommandBuilder.stopTask(fileNames); + System.out.println(" killTask: "+killTask); int exitCode = ProcessUtils.executeCommand(killTask,null); if(exitCode != 0 ){ - exitCode = linuxKill(fileName); + exitCode = linuxKill(fileNames); } return exitCode; } - private int linuxKill(String fileName){ - List containerIds = podMonitor.getContainerIds(fileName); + private int linuxKill(List files){ + System.out.println("########### linuxKill ##########, fileName: "+files); + List containerIds = podMonitor.getContainerIds(files); List pids = new ArrayList(); for(String containerId:containerIds){ + System.out.println(" containerId: "+containerId); ContainerDetails details = podMonitor.getContainerDetails(containerId); + System.out.println(" details: "+details); int pid = details.getPid(); if(pid != 0){ pids.add(pid); @@ -130,7 +205,9 @@ private int linuxKill(String fileName){ } int exitCode = 1; if(pids.size() > 0){ + System.out.println("pids: "+pids.toString()); String command = CommandBuilder.linuxKill(pids); + System.out.println(" command: "+command); exitCode = ProcessUtils.executeCommand(command, null); } return exitCode; @@ -179,7 +256,57 @@ public void reregistered(ExecutorDriver executorDriver, SlaveInfo slaveInfo) { @Override public void shutdown(ExecutorDriver executorDriver) { log.debug("shutting down executor"); + shutdownPlugin(); suicide(null, 0); } + + private List updateYamlFiles(List paths, ExecutorInfo executorInfo,TaskInfo taskInfo) throws IOException { + List newPath = new ArrayList<>(); + for (String file: paths) { + Map>> rootYaml = readFromFile(file); + System.out.println(" rootYaml dump: " + rootYaml.toString()); + Map>> updatedYaml = composeRewriteHelper.updateYaml(rootYaml, taskInfo, executorInfo); + System.out.println(" updatedYaml dump: "+updatedYaml.toString()); + String outputFileName = getOutputFileName(file); + System.out.println("outputFileName: "+outputFileName); + writeToFile(outputFileName, updatedYaml); + newPath.add(outputFileName); + } + System.out.println("updated Paths: "+newPath.toString()); + return newPath; + } + + private String getOutputFileName(String path){ + if(path != null ){ + StringBuffer buf = new StringBuffer(path.trim()); + buf.append(GENERATED_YAML_FILE_NAME); + return buf.toString(); + } + return null; + } + + private File writeToFile(String fileName,Map>> updatedRootYaml) throws IOException,FileNotFoundException { + File file = new File(fileName); + FileWriter fileWriter = new FileWriter(file); + DumperOptions options=new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yaml = new Yaml(options); + yaml.dump(updatedRootYaml,fileWriter); + fileWriter.flush(); + fileWriter.close(); + return file; + } + + private Map>> readFromFile(String path) throws FileNotFoundException,IOException{ + FileReader fileReader = new FileReader(new File(path)); + @SuppressWarnings("unchecked") + DumperOptions options=new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yaml = new Yaml(options); + Map>> yamlMap = (Map>>)yaml.load(fileReader); + fileReader.close(); + return yamlMap; + } + } diff --git a/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java b/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java index a3dd68e..37772d2 100644 --- a/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java +++ b/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java @@ -7,7 +7,7 @@ import dagger.Component; @Singleton -@Component(modules = { FileFetcherModule.class,ExecutorModule.class}) +@Component(modules = { ComposeFileListModule.class,ExecutorModule.class}) public interface ExecutorComponent { Executor getExecutor(); diff --git a/src/main/java/com/paypal/mesos/executor/ExecutorModule.java b/src/main/java/com/paypal/mesos/executor/ExecutorModule.java index eef1584..ccda793 100644 --- a/src/main/java/com/paypal/mesos/executor/ExecutorModule.java +++ b/src/main/java/com/paypal/mesos/executor/ExecutorModule.java @@ -2,19 +2,23 @@ import javax.inject.Singleton; +import com.paypal.mesos.executor.compose.ComposeRewriteHelper; import org.apache.mesos.Executor; -import com.paypal.mesos.executor.fetcher.FileFetcher; +import com.paypal.mesos.executor.compose.ComposeFileList; import com.paypal.mesos.executor.monitoring.ComposeMonitor; import dagger.Module; import dagger.Provides; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; @Module public class ExecutorModule { - @Provides Executor provideDockerComposeExecutor(FileFetcher fileFetcher,DockerComposeProcessObserver processObserver,ComposeMonitor composeMonitor){ - return new DockerComposeExecutor(fileFetcher,processObserver,composeMonitor); + @Provides Executor provideDockerComposeExecutor(ComposeFileList fileFetcher,DockerComposeProcessObserver processObserver, + ComposeMonitor composeMonitor, ComposeRewriteHelper helper){ + return new DockerComposeExecutor(fileFetcher,processObserver,composeMonitor, helper); } @Provides @Singleton ComposeMonitor provideComposeMonitor(){ @@ -25,4 +29,15 @@ public class ExecutorModule { return new DockerComposeProcessObserver(); } + @Provides @Singleton + ComposeRewriteHelper provideComposeRewriteHelper() { + return new ComposeRewriteHelper(); + } + + @Provides @Singleton Yaml provideYaml(){ + DumperOptions options=new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + return new Yaml(options); + } + } diff --git a/src/main/java/com/paypal/mesos/executor/FileFetcherModule.java b/src/main/java/com/paypal/mesos/executor/FileFetcherModule.java deleted file mode 100644 index ab70c1c..0000000 --- a/src/main/java/com/paypal/mesos/executor/FileFetcherModule.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.paypal.mesos.executor; - -import javax.inject.Singleton; - -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; - -import com.paypal.mesos.executor.fetcher.DockerComposeFileFetcher; -import com.paypal.mesos.executor.fetcher.DockerRewriteHelper; -import com.paypal.mesos.executor.fetcher.FileFetcher; - -import dagger.Module; -import dagger.Provides; - -@Module -public class FileFetcherModule { - - @Provides @Singleton FileFetcher provideComposeFileFetcher(DockerRewriteHelper helper,Yaml yaml){ - return new DockerComposeFileFetcher(helper,yaml); - } - - @Provides @Singleton DockerRewriteHelper provideRewriteHelper(){ - return new DockerRewriteHelper(); - } - - @Provides @Singleton Yaml provideYaml(){ - DumperOptions options=new DumperOptions(); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - return new Yaml(options); - } - -} diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java new file mode 100644 index 0000000..f1f3971 --- /dev/null +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java @@ -0,0 +1,12 @@ +package com.paypal.mesos.executor.compose; + +import java.io.IOException; +import java.util.List; + +import org.apache.mesos.Protos.TaskInfo; + +public interface ComposeFileList { + + List getFile(TaskInfo taskInfo) throws IOException; + +} diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java new file mode 100644 index 0000000..cf9894e --- /dev/null +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java @@ -0,0 +1,134 @@ +package com.paypal.mesos.executor.compose; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.log4j.Logger; +import org.apache.mesos.Protos.Label; +import org.apache.mesos.Protos.Labels; +import org.apache.mesos.Protos.TaskInfo; + + +public class ComposeFileListImpl implements ComposeFileList{ + + private static final Logger log = Logger.getLogger(ComposeFileListImpl.class); + +// private static final String GENERATED_YAML_FILE_NAME = "-generated.yml"; + private static final String FILE_DELIMITER = ","; +// +// private ComposeRewriteHelper helper; +// +// private Yaml yaml; + +// ExecutorInfo info; + +// public ComposeFileListImpl(DockerRewriteHelper helper,Yaml yaml) { +// this.helper = helper; +// this.yaml = yaml; +// } + + @Inject + public ComposeFileListImpl() { + + } + + @Override + public List getFile(TaskInfo taskInfo) throws FileNotFoundException,IOException{ + System.out.println("############## STARTING DockerComposeFileFetcher.getFile ##############"); + + List paths = getFileName(taskInfo); + System.out.println(" paths: " + paths); + validateFiles(paths); + + return paths; + // Refactored code .. move yaml update logic in a method.. +// return updateYamlFiles(paths, executorInfo, taskInfo); + } + +// private List updateYamlFiles(List paths, ExecutorInfo executorInfo,TaskInfo taskInfo) throws IOException { +// List newPath = new ArrayList<>(); +// for (String file: paths) { +// Map>> rootYaml = readFromFile(file); +// System.out.println(" rootYaml dump: " + rootYaml.toString()); +// Map>> updatedYaml = helper.updateYaml(rootYaml, taskInfo, executorInfo); +// System.out.println(" updatedYaml dump: "+updatedYaml.toString()); +// String outputFileName = getOutputFileName(file); +// System.out.println("outputFileName: "+outputFileName); +// writeToFile(outputFileName, updatedYaml); +// newPath.add(outputFileName); +// } +// return newPath; +// } + +/* private String getOutputFileName(String path){ + if(path != null ){ + StringBuffer buf = new StringBuffer(path.trim()); + buf.append(GENERATED_YAML_FILE_NAME); + return buf.toString(); + } + return null; + }*/ + + //TODO figure out a way to lookup a map instead of iteration + private List getFileName(TaskInfo taskInfo){ + + Labels labels = taskInfo.getLabels(); + for(Label label:labels.getLabelsList()){ + if("fileName".equals(label.getKey())){ + return Arrays.asList(label.getValue().split(FILE_DELIMITER)); + } + } + log.warn("error reading fileName from taskInfo"); + return null; + } + +/* + private File writeToFile(String fileName,Map>> updatedRootYaml) throws IOException,FileNotFoundException{ + File file = new File(fileName); + FileWriter fileWriter = new FileWriter(file); + yaml.dump(updatedRootYaml,fileWriter); + fileWriter.flush(); + fileWriter.close(); + return file; + } + + private Map>> readFromFile(String path) throws FileNotFoundException,IOException{ + FileReader fileReader = new FileReader(new File(path)); + @SuppressWarnings("unchecked") + Map>> yamlMap = (Map>>)yaml.load(fileReader); + fileReader.close(); + return yamlMap; + } +*/ + + + private void validateFiles(List paths) throws FileNotFoundException{ + System.out.println(" ########### validateFiles ###########"); + System.out.println(" paths: "+paths.toString()); + if(paths == null || paths.size() == 0 ){ + throw new FileNotFoundException("empty .yml/.yaml file list @"); + } + for (String name: paths) { + File file = new File(name); + if (!file.isFile() || !(name.endsWith(".yml") || name.endsWith(".yaml"))){ + throw new FileNotFoundException("No .yml/.yaml file found @"+name); + } + } + return; + } + + private boolean fileExists(List paths){ + for (String name: paths) { + File file = new File(name); + String fileName = file.getName(); + return file.isFile() && (fileName.endsWith(".yml") || fileName.endsWith(".yaml")); + } + return true; + } +} + diff --git a/src/main/java/com/paypal/mesos/executor/fetcher/DockerRewriteHelper.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java similarity index 84% rename from src/main/java/com/paypal/mesos/executor/fetcher/DockerRewriteHelper.java rename to src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java index 2edd09e..3adb754 100644 --- a/src/main/java/com/paypal/mesos/executor/fetcher/DockerRewriteHelper.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java @@ -1,4 +1,4 @@ -package com.paypal.mesos.executor.fetcher; +package com.paypal.mesos.executor.compose; import java.util.ArrayList; import java.util.HashMap; @@ -14,7 +14,7 @@ import org.apache.mesos.Protos.Value.Range; import org.apache.mesos.Protos.Value.Ranges; -public class DockerRewriteHelper { +public class ComposeRewriteHelper { private static final String CONTAINER_NAME = "container_name"; @@ -27,15 +27,21 @@ public class DockerRewriteHelper { private static final String SERVICES = "services"; public Map>> updateYaml(Map>> yamlMap,TaskInfo taskInfo,ExecutorInfo executorInfo){ + System.out.println(" ############ STARTING updateYaml ################"); if(yamlMap == null || yamlMap.isEmpty()){ return null; } Map>> resultantContainerMap = new HashMap>>(); resultantContainerMap.putAll(yamlMap); String taskId = taskInfo.getTaskId().getValue(); + System.out.println(" taskId: " + taskId); + Iterator portIterator = getPortMappingIterator(taskInfo); + // System.out.println(" portIterator: " + portIterator.toString());// this doesnt help String executorId = executorInfo.getExecutorId().getValue(); + System.out.println(" executorId: " + executorId); Map> services = yamlMap.get(SERVICES); + System.out.println(" services map: " + services.toString()); Map> resultantServicesMap = new HashMap>(); for(Map.Entry> containerEntry:services.entrySet()){ @@ -52,8 +58,12 @@ public Map>> updateYaml(Map updateContainerValue(String executorId,String taskId,Map containerDetails,Iterator portIterator){ + System.out.println(" ################## Starting updateContainerValue ############### "); + System.out.println(" executorId: " + executorId + " taskId: " + taskId + " containerDetails: " + containerDetails.toString() + " portIterator: " + portIterator.toString()); + if(containerDetails.containsKey(CONTAINER_NAME)){ String containerValue = prefixTaskId(taskId,String.valueOf(containerDetails.get(CONTAINER_NAME))); + System.out.println("updated ContainerName: "+containerValue); containerDetails.put(CONTAINER_NAME, containerValue); } @@ -62,6 +72,7 @@ private Map updateContainerValue(String executorId,String taskId, String networkValueString = String.valueOf(networkValue); String [] split = networkValueString.split(":"); String containerName = split[split.length-1]; + System.out.println("updated network: " + "service:" + prefixTaskId(taskId, containerName)); containerDetails.put(NETWORK, "service:"+prefixTaskId(taskId, containerName)); } System.out.println("In update container values"); @@ -75,6 +86,7 @@ private Map updateContainerValue(String executorId,String taskId, updatedLinks.add(prefixTaskId(taskId, link)+":"+link); System.out.println(prefixTaskId(taskId, link)); } + System.out.println(" updatedLinks: "+updatedLinks); containerDetails.put(LINKS, updatedLinks); } @@ -86,6 +98,7 @@ private Map updateContainerValue(String executorId,String taskId, for(String link:links){ updatedLinks.add(prefixTaskId(taskId, link)); } + System.out.println("updated Links: "+updatedLinks); containerDetails.put(DEPENDS_ON, updatedLinks); } @@ -97,6 +110,7 @@ private Map updateContainerValue(String executorId,String taskId, for(String volumeFrom:volumesFrom){ updatedVolumesFrom.add(prefixTaskId(taskId, volumeFrom)); } + System.out.println(" updated Volumes: "+updatedVolumesFrom); containerDetails.put(VOLUMES_FROM, updatedVolumesFrom); } @@ -109,6 +123,7 @@ private Map updateContainerValue(String executorId,String taskId, String replacedPort = replacePort(portString,portIterator); updatedPorts.add(replacedPort); } + System.out.println(" updatedPorts: "+updatedPorts); containerDetails.put(PORTS, updatedPorts); } @@ -119,6 +134,7 @@ private Map updateContainerValue(String executorId,String taskId, } taskIdLabel.put("taskId", taskId); taskIdLabel.put("executorId",executorId); + System.out.println(" updated taskIdLabel: "+ taskIdLabel); containerDetails.put(LABELS, taskIdLabel); return containerDetails; diff --git a/src/main/java/com/paypal/mesos/executor/fetcher/DockerComposeFileFetcher.java b/src/main/java/com/paypal/mesos/executor/fetcher/DockerComposeFileFetcher.java deleted file mode 100644 index 7314b03..0000000 --- a/src/main/java/com/paypal/mesos/executor/fetcher/DockerComposeFileFetcher.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.paypal.mesos.executor.fetcher; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.util.Map; - -import javax.inject.Inject; - -import org.apache.log4j.Logger; -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.Label; -import org.apache.mesos.Protos.Labels; -import org.apache.mesos.Protos.TaskInfo; -import org.yaml.snakeyaml.Yaml; - - -public class DockerComposeFileFetcher implements FileFetcher{ - - private static final Logger log = Logger.getLogger(DockerComposeFileFetcher.class); - - private static final String GENERATED_YAML_FILE_NAME = "docker-compose-generated.yml"; - - private DockerRewriteHelper helper; - - private Yaml yaml; - - ExecutorInfo info; - - @Inject - public DockerComposeFileFetcher(DockerRewriteHelper helper,Yaml yaml) { - this.helper = helper; - this.yaml = yaml; - } - - @Override - public File getFile(ExecutorInfo executorInfo,TaskInfo taskInfo) throws FileNotFoundException,IOException{ - String path = getFileName(taskInfo); - validateFile(path); - Map>> rootYaml = readFromFile(path); - Map>> updatedYaml = helper.updateYaml(rootYaml,taskInfo,executorInfo); - String outputFileName = getOutputFileName(path); - return writeToFile(outputFileName,updatedYaml); - } - - private String getOutputFileName(String path){ - if(path != null && path.split("/").length > 1){ - String [] tokens = path.split("/"); - String result = tokens[0]; - for(int i=1;i>> updatedRootYaml) throws IOException,FileNotFoundException{ - File file = new File(fileName); - FileWriter fileWriter = new FileWriter(file); - yaml.dump(updatedRootYaml,fileWriter); - fileWriter.flush(); - fileWriter.close(); - return file; - } - - private Map>> readFromFile(String path) throws FileNotFoundException,IOException{ - FileReader fileReader = new FileReader(new File(path)); - @SuppressWarnings("unchecked") - Map>> yamlMap = (Map>>)yaml.load(fileReader); - fileReader.close(); - return yamlMap; - } - - - private void validateFile(String path) throws FileNotFoundException{ - if(path == null || path.length() == 0 || !fileExists(path)){ - throw new FileNotFoundException("No .yml/.yaml file found @"+path); - } - } - - private boolean fileExists(String path){ - File file = new File(path); - String fileName = file.getName(); - return file.isFile() && (fileName.endsWith(".yml") || fileName.endsWith(".yaml")); - } - -} - diff --git a/src/main/java/com/paypal/mesos/executor/fetcher/FileFetcher.java b/src/main/java/com/paypal/mesos/executor/fetcher/FileFetcher.java deleted file mode 100644 index 035c910..0000000 --- a/src/main/java/com/paypal/mesos/executor/fetcher/FileFetcher.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.paypal.mesos.executor.fetcher; - -import java.io.File; -import java.io.IOException; - -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.TaskInfo; - -public interface FileFetcher { - - File getFile(ExecutorInfo executorInfo,TaskInfo taskInfo) throws IOException; - -} diff --git a/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java b/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java index fbe9bef..0debd69 100644 --- a/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java +++ b/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java @@ -2,6 +2,7 @@ import java.io.ByteArrayOutputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.StringTokenizer; import java.util.concurrent.TimeUnit; @@ -33,24 +34,25 @@ public Subscription subscribeToChanges(Action1 action){ return monitor.subscribe(action); } - public void startMonitoring(final String fileName){ - log.info("start montioring is called:"+fileName); + public void startMonitoring(final List fileNames){ + log.info("start montioring is called:"+fileNames); Observable.interval(Config.POD_MONITOR_INTERVAL, TimeUnit.MILLISECONDS).subscribe(new Observer() { @Override public void onCompleted() { - log.info("monitor thread on completed :completed monitoring compose for file:"+fileName); + log.info("monitor thread on completed :completed monitoring compose for file:"+fileNames); } @Override public void onError(Throwable e) { - log.error("monitor thread on error: encountred an error monitoring:"+fileName,e); + log.error("monitor thread on error: encountred an error monitoring:"+fileNames,e); monitor.onNext(1); } @Override public void onNext(Long t) { - List containerIds = getContainerIds(fileName); + List containerIds = getContainerIds(fileNames); + System.out.println(Arrays.toString(containerIds.toArray())); if(containerIds != null){ for(String containerId:containerIds){ ContainerDetails details = getContainerDetails(containerId); @@ -67,9 +69,9 @@ public void onNext(Long t) { } - public List getContainerIds(String fileName){ + public List getContainerIds(List fileNames){ List containerIds = new ArrayList(); - String listConatinerIdsCommand = CommandBuilder.getContainerIds(fileName); + String listConatinerIdsCommand = CommandBuilder.getContainerIds(fileNames); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); int exitCode = ProcessUtils.executeCommand(listConatinerIdsCommand, null, outputStream, null, null); if(exitCode == 0){ diff --git a/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java new file mode 100644 index 0000000..a026d26 --- /dev/null +++ b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java @@ -0,0 +1,21 @@ +package com.paypal.mesos.executor.pluginapi; + +import org.apache.mesos.ExecutorDriver; +import org.apache.mesos.Protos; +import ro.fortsoft.pf4j.ExtensionPoint; + +import java.io.IOException; +import java.util.List; + +/** + * Created by kkrishna on 7/18/16. + */ +public interface ComposeExecutorPlugin extends ExtensionPoint { + + public List launchTask(ExecutorDriver executorDriver, Protos.TaskInfo taskInfo); + + public void shutdown(); + + public List getComposeFiles(); + +} diff --git a/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPluginImpl.java b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPluginImpl.java new file mode 100644 index 0000000..537bcd5 --- /dev/null +++ b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPluginImpl.java @@ -0,0 +1,51 @@ +package com.paypal.mesos.executor.pluginapi; + +import com.paypal.mesos.executor.compose.ComposeFileList; +import com.paypal.mesos.executor.compose.ComposeFileListImpl; +import org.apache.mesos.ExecutorDriver; +import org.apache.mesos.Protos; +import ro.fortsoft.pf4j.Extension; + +import java.io.IOException; +import java.util.List; + +/** + * Created by kkrishna on 7/18/16. + */ +@Extension +public class ComposeExecutorPluginImpl implements ComposeExecutorPlugin { + @Override + public List launchTask(ExecutorDriver executorDriver, Protos.TaskInfo taskInfo) { + System.out.println(" ######### ComposeExecutorPlugin launchTask #############"); + System.out.println(" ExecutorDriver: " + executorDriver.toString()); + System.out.println(" taskInfo: "+ taskInfo.toString()); + ComposeFileList composeFiles = new ComposeFileListImpl(); + Protos.TaskID taskId = taskInfo.getTaskId(); + + try { + return composeFiles.getFile(taskInfo); + } catch (IOException e) { + e.printStackTrace(); + Protos.TaskStatus taskStatus = Protos.TaskStatus.newBuilder().setTaskId(taskId).setState(Protos.TaskState.TASK_FAILED).build(); + executorDriver.sendStatusUpdate(taskStatus); + } + +// Protos.Labels labels = taskInfo.getLabels(); +// System.out.println("taskInfo labels: " + labels.toString()); +// labels.toBuilder().addLabels(Protos.Label.newBuilder().setKey("testkey").setValue("testvalue")).build(); +// taskInfo.toBuilder().setLabels(labels).build(); +// +// System.out.println(" taskInfo: " + taskInfo.toString()); + return null; + } + + @Override + public void shutdown() { + System.out.println(" ############## ComposeExecutorPlugin Shutdown ############"); + } + + @Override + public List getComposeFiles() { + return null; + } +} diff --git a/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java b/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java index 63d3c83..fd9b886 100644 --- a/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java +++ b/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java @@ -21,6 +21,7 @@ public static int executeCommand(String command,ExecuteWatchdog watchdog) { } public static int executeCommand(String command,ExecuteWatchdog watchdog,OutputStream outputStream,OutputStream errorStream,InputStream inputStream){ + System.out.println(" command: "+command); CommandLine cmdLine = CommandLine.parse(command); DefaultExecutor executor = new DefaultExecutor(); if(outputStream == null){ @@ -41,6 +42,8 @@ public static int executeCommand(String command,ExecuteWatchdog watchdog,OutputS exitValue = 1; log.error("error executing command", e); } + + System.out.println(" output: "+outputStream.toString()); return exitValue; } From 2b88f189da0a6ece49f0d4d328602cf9305b2538 Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Fri, 22 Jul 2016 15:29:39 -0700 Subject: [PATCH 02/15] Adding all mesos labels to each container. Reformatted some long function signatures --- .../compose/ComposeRewriteHelper.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java index 3adb754..a9ce197 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java @@ -8,6 +8,7 @@ import java.util.Map; import java.util.NoSuchElementException; +import org.apache.mesos.Protos; import org.apache.mesos.Protos.ExecutorInfo; import org.apache.mesos.Protos.Resource; import org.apache.mesos.Protos.TaskInfo; @@ -26,7 +27,10 @@ public class ComposeRewriteHelper { private static final String LABELS = "labels"; private static final String SERVICES = "services"; - public Map>> updateYaml(Map>> yamlMap,TaskInfo taskInfo,ExecutorInfo executorInfo){ + public Map>> updateYaml(Map>> yamlMap, + TaskInfo taskInfo, + ExecutorInfo executorInfo){ + System.out.println(" ############ STARTING updateYaml ################"); if(yamlMap == null || yamlMap.isEmpty()){ return null; @@ -47,7 +51,10 @@ public Map>> updateYaml(Map containerValue = containerEntry.getValue(); - Map updatedContainerValues = updateContainerValue(executorId,taskId,containerValue,portIterator); + Map updatedContainerValues = updateContainerValue(executorId, + taskInfo, + containerValue, + portIterator); String updatedKey = prefixTaskId(taskId, key); resultantServicesMap.put(updatedKey,updatedContainerValues); } @@ -56,8 +63,12 @@ public Map>> updateYaml(Map updateContainerValue(String executorId,String taskId,Map containerDetails,Iterator portIterator){ + private Map updateContainerValue(String executorId, + TaskInfo taskInfo, + Map containerDetails, + Iterator portIterator){ + String taskId = taskInfo.getTaskId().getValue(); System.out.println(" ################## Starting updateContainerValue ############### "); System.out.println(" executorId: " + executorId + " taskId: " + taskId + " containerDetails: " + containerDetails.toString() + " portIterator: " + portIterator.toString()); @@ -134,6 +145,11 @@ private Map updateContainerValue(String executorId,String taskId, } taskIdLabel.put("taskId", taskId); taskIdLabel.put("executorId",executorId); + + for(Protos.Label l : taskInfo.getLabels().getLabelsList()) { + taskIdLabel.put(l.getKey(), l.getValue()); + } + System.out.println(" updated taskIdLabel: "+ taskIdLabel); containerDetails.put(LABELS, taskIdLabel); From 67eef22b53220880fc512a921e950e258c6cdf21 Mon Sep 17 00:00:00 2001 From: Kumar Krishna Date: Wed, 3 Aug 2016 10:47:06 -0700 Subject: [PATCH 03/15] added unit test , logging .. --- pom.xml | 14 +- .../paypal/mesos/executor/CommandBuilder.java | 96 +-- .../mesos/executor/ComposeFileListModule.java | 8 +- .../mesos/executor/DockerComposeExecutor.java | 550 +++++++++--------- .../DockerComposeProcessObserver.java | 56 +- .../mesos/executor/ExecutorComponent.java | 6 +- .../paypal/mesos/executor/ExecutorModule.java | 54 +- .../executor/compose/ComposeFileList.java | 4 +- .../executor/compose/ComposeFileListImpl.java | 175 ++---- .../compose/ComposeRewriteHelper.java | 363 ++++++------ .../executor/monitoring/ComposeMonitor.java | 203 +++---- .../executor/monitoring/ContainerDetails.java | 106 ++-- .../pluginapi/ComposeExecutorPlugin.java | 10 +- .../pluginapi/ComposeExecutorPluginImpl.java | 38 +- .../mesos/executor/utils/ProcessUtils.java | 95 ++- .../compose/ComposeRewriteHelperTest.java | 70 +++ src/test/resources/docker-compose.yml | 12 + 17 files changed, 944 insertions(+), 916 deletions(-) create mode 100644 src/test/java/com/paypal/mesos/executor/compose/ComposeRewriteHelperTest.java create mode 100644 src/test/resources/docker-compose.yml diff --git a/pom.xml b/pom.xml index 9b2417d..05f4c11 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,12 @@ pf4j 0.13.1 + + junit + junit + 4.11 + test + @@ -127,14 +133,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 3.1 - - true - - diff --git a/src/main/java/com/paypal/mesos/executor/CommandBuilder.java b/src/main/java/com/paypal/mesos/executor/CommandBuilder.java index ecad431..20c5755 100644 --- a/src/main/java/com/paypal/mesos/executor/CommandBuilder.java +++ b/src/main/java/com/paypal/mesos/executor/CommandBuilder.java @@ -7,65 +7,65 @@ public class CommandBuilder { - public static String launchTask(List files){ - StringBuffer buf = new StringBuffer("docker-compose "); - for (String file : files) { - buf.append(" -f "+file); - } - buf.append(" up"); - return buf.toString(); - //return "docker-compose -f "+fileName+" up"; - } + public static String launchTask(List files) { + StringBuffer buf = new StringBuffer("docker-compose "); + for (String file : files) { + buf.append(" -f " + file); + } + buf.append(" up"); + return buf.toString(); + //return "docker-compose -f "+fileName+" up"; + } - public static String pullImages(List files){ - StringBuffer buf = new StringBuffer("docker-compose "); - for (String file : files) { - buf.append(" -f "+file); - } - buf.append(" pull"); + public static String pullImages(List files) { + StringBuffer buf = new StringBuffer("docker-compose "); + for (String file : files) { + buf.append(" -f " + file); + } + buf.append(" pull"); - if(Config.IGNORE_PULL_FAILURES){ - buf.append(" --ignore-pull-failures"); - } - return buf.toString(); + if (Config.IGNORE_PULL_FAILURES) { + buf.append(" --ignore-pull-failures"); + } + return buf.toString(); // String command = "docker-compose -f "+fileName+" pull"; // if(Config.IGNORE_PULL_FAILURES){ // command = command+" --ignore-pull-failures"; // } // return command; - } + } - public static String stopTask(List files){ - StringBuffer buf = new StringBuffer("docker-compose "); - for (String file : files) { - buf.append(" -f "+file); - } - buf.append(" stop"); - return buf.toString(); - //return "docker-compose -f "+fileName+" stop "; - } + public static String stopTask(List files) { + StringBuffer buf = new StringBuffer("docker-compose "); + for (String file : files) { + buf.append(" -f " + file); + } + buf.append(" stop"); + return buf.toString(); + //return "docker-compose -f "+fileName+" stop "; + } - public static String getContainerIds(List files){ - StringBuffer buf = new StringBuffer("docker-compose "); - for (String file : files) { - buf.append(" -f "+file); - } - buf.append(" ps -q"); - return buf.toString(); - //return "docker-compose -f "+fileName + " ps -q"; - } + public static String getContainerIds(List files) { + StringBuffer buf = new StringBuffer("docker-compose "); + for (String file : files) { + buf.append(" -f " + file); + } + buf.append(" ps -q"); + return buf.toString(); + //return "docker-compose -f "+fileName + " ps -q"; + } - public static String linuxKill(List pids){ - StringBuilder processIds = new StringBuilder(); - for(int pid:pids){ - processIds.append(pid).append(" "); - } - return "sudo kill -9 "+processIds.toString(); - } + public static String linuxKill(List pids) { + StringBuilder processIds = new StringBuilder(); + for (int pid : pids) { + processIds.append(pid).append(" "); + } + return "sudo kill -9 " + processIds.toString(); + } - public static String getContainerDetails(String containerId){ - return "docker inspect --format='{{.State.Pid}},{{.State.ExitCode}},{{.State.Running}},{{.RestartCount}},{{.HostConfig.RestartPolicy.MaximumRetryCount}}' "+containerId; - } + public static String getContainerDetails(String containerId) { + return "docker inspect --format='{{.State.Pid}},{{.State.ExitCode}},{{.State.Running}},{{.RestartCount}},{{.HostConfig.RestartPolicy.MaximumRetryCount}}' " + containerId; + } } diff --git a/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java b/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java index 39e2b88..f321e4f 100644 --- a/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java +++ b/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java @@ -11,8 +11,10 @@ @Module public class ComposeFileListModule { - @Provides @Singleton ComposeFileList provideComposeFileList(){ - return new ComposeFileListImpl(); - } + @Provides + @Singleton + ComposeFileList provideComposeFileList() { + return new ComposeFileListImpl(); + } } diff --git a/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java b/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java index 2a0f6a7..4b1ce76 100644 --- a/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java +++ b/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java @@ -34,279 +34,281 @@ import com.paypal.mesos.executor.monitoring.ContainerDetails; import com.paypal.mesos.executor.utils.ProcessUtils; -public class DockerComposeExecutor implements Executor{ - - private static final Logger log = Logger.getLogger(DockerComposeExecutor.class); - private static final String GENERATED_YAML_FILE_NAME = "-generated.yml"; - - - private ComposeFileList fileFetcher; - - private List fileNames; - - private ExecutorDriver executorDriver; - - private DockerComposeProcessObserver processObserver; - - private ExecutorInfo executorInfo; - - private ComposeMonitor podMonitor; - - private ComposeRewriteHelper composeRewriteHelper; - - final PluginManager pluginManager = new DefaultPluginManager(); - - private boolean pluginEnabled = false; - - @Inject - public DockerComposeExecutor(ComposeFileList fileFetcher,DockerComposeProcessObserver processObserver, - ComposeMonitor podMonitor, ComposeRewriteHelper helper){ - this.fileFetcher = fileFetcher; - this.processObserver = processObserver; - this.podMonitor = podMonitor; - this.composeRewriteHelper = helper; - //load/start plugins needed only if plugin is loaded from separate jar -// this.pluginManager.loadPlugins(); -// this.pluginManager.startPlugins(); - } - - @Override - public void launchTask(ExecutorDriver executorDriver, TaskInfo taskInfo) { - System.out.println(" ############ Launch Task START #############"); - System.out.println(" taskInfo: "+taskInfo.toString()); - System.out.println(" executorDriver: "+executorDriver.toString()); - TaskID taskId = taskInfo.getTaskId(); - - - System.out.println(" taskId: " + taskId.getValue()); - - processObserver.init(this, taskId); - sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_STARTING); - try { - //this.fileNames = fileFetcher.getFile(taskInfo); - this.fileNames = executePlugin(executorDriver, taskInfo); - if (this.fileNames == null || this.fileNames.size() == 0) { - sendTaskStatusUpdate(executorDriver,taskId,TaskState.TASK_FAILED); - return; - } - - this.fileNames = updateYamlFiles(this.fileNames, executorInfo, taskInfo); - //this.fileNames = file.getAbsolutePath(); - System.out.println(" fileName: "+this.fileNames); - podMonitor.subscribeToChanges(new Action1() { - @Override - public void call(Integer exitCode) { - suicide(taskId, exitCode); - } - }); - podMonitor.startMonitoring(this.fileNames); - updateImagesAndStartCompose(taskId); - sendTaskStatusUpdate(executorDriver,taskId,TaskState.TASK_RUNNING); - }catch (Exception e) { - log.error("exception while launching process",e); - sendTaskStatusUpdate(executorDriver,taskId,TaskState.TASK_FAILED); - } - } - - private List executePlugin(ExecutorDriver executorDriver, TaskInfo taskInfo) { - if (this.pluginManager != null) { - List plugins = this.pluginManager.getExtensions(ComposeExecutorPlugin.class); - System.out.println(String.format("Found %d extensions for extension point '%s'", plugins.size(), ComposeExecutorPlugin.class.getName())); - if (plugins.size() > 1) { - log.error(" more than one extension implementation: "+plugins.toString()); - return null; - } - //check to ensure that only a single plugin exists.. - for (ComposeExecutorPlugin plugin : plugins) { - return plugin.launchTask(executorDriver, taskInfo); - } - }else { - log.error(" pluginManager NULL "); - } - return null; - } - - private void shutdownPlugin() { - if (this.pluginManager != null) { - List plugins = this.pluginManager.getExtensions(ComposeExecutorPlugin.class); - System.out.println(String.format("Found %d extensions for extension point '%s'", plugins.size(), ComposeExecutorPlugin.class.getName())); - for (ComposeExecutorPlugin plugin : plugins) { - plugin.shutdown(); - } - } - } - - private void updateImagesAndStartCompose(TaskID taskId){ - Observable.create(new Observable.OnSubscribe() { - @Override - public void call(Subscriber subscriber) { - String pullCommand = CommandBuilder.pullImages(fileNames); - System.out.println(" pullCommand: "+pullCommand); - int imageUpdateExitCode = ProcessUtils.executeCommand(pullCommand, null); - if(imageUpdateExitCode != 0){ - log.error("unable to pull updated images trying to bring the pod up with existing images"); - } - String launchCommand = CommandBuilder.launchTask(fileNames); - System.out.println(" launchCommand: "+ launchCommand); - int exitCode = ProcessUtils.executeCommand(launchCommand, null); - subscriber.onNext(exitCode); - subscriber.onCompleted(); - } - }).subscribeOn(Schedulers.newThread()).observeOn(Schedulers.newThread()).subscribe(processObserver); - } - - public void suicide(TaskID taskId,int exitCode){ - System.out.println(" ############## in suicide #######"); - System.out.println("taskId: "+taskId.toString() + " exitCode: "+exitCode); - int stopContainers = cleanUp(); - - if (this.pluginManager != null) { - this.pluginManager.stopPlugins(); - } - - if(exitCode == 0 && stopContainers == 0){ - sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FINISHED); - System.exit(0); - }else{ - sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FAILED); - System.exit(1); - } - } - - - /** - * stop docker-compose - * force remove docker images - * @return killing compose and removing all images are successful - */ - private int cleanUp(){ - System.out.println(" ############## Inside cleanUp ###########, fileName: "+fileNames); - String killTask = CommandBuilder.stopTask(fileNames); - System.out.println(" killTask: "+killTask); - int exitCode = ProcessUtils.executeCommand(killTask,null); - if(exitCode != 0 ){ - exitCode = linuxKill(fileNames); - } - return exitCode; - } - - private int linuxKill(List files){ - System.out.println("########### linuxKill ##########, fileName: "+files); - List containerIds = podMonitor.getContainerIds(files); - List pids = new ArrayList(); - for(String containerId:containerIds){ - System.out.println(" containerId: "+containerId); - ContainerDetails details = podMonitor.getContainerDetails(containerId); - System.out.println(" details: "+details); - int pid = details.getPid(); - if(pid != 0){ - pids.add(pid); - } - } - int exitCode = 1; - if(pids.size() > 0){ - System.out.println("pids: "+pids.toString()); - String command = CommandBuilder.linuxKill(pids); - System.out.println(" command: "+command); - exitCode = ProcessUtils.executeCommand(command, null); - } - return exitCode; - } - - private void sendTaskStatusUpdate(ExecutorDriver executorDriver,TaskID taskId,TaskState taskState){ - TaskStatus taskStatus = TaskStatus.newBuilder().setTaskId(taskId).setState(taskState).build(); - executorDriver.sendStatusUpdate(taskStatus); - } - - @Override - public void killTask(ExecutorDriver executorDriver, TaskID taskId) { - log.info("kill task called for taskId:"+taskId.getValue()); - suicide(taskId,0); - } - - @Override - public void disconnected(ExecutorDriver executorDriver) { - log.debug("executor disconnected"); - } - - @Override - public void error(ExecutorDriver executorDriver, String errorMessage) { - log.error("executor received an error message:"+errorMessage); - } - - @Override - public void frameworkMessage(ExecutorDriver arg0, byte[] arg1) { - log.debug("received framework message"); - } - - @Override - public void registered(ExecutorDriver executorDriver, ExecutorInfo executorInfo, - FrameworkInfo frameworkInfo, SlaveInfo slaveInfo) { - log.debug("executor registered with framework:"+frameworkInfo.getName()+":on slave:"+slaveInfo.getHostname()); - this.executorInfo = executorInfo; - this.executorDriver = executorDriver; - } - - @Override - public void reregistered(ExecutorDriver executorDriver, SlaveInfo slaveInfo) { - log.debug("executor reregistered on slave:"+slaveInfo.getHostname()); - this.executorDriver = executorDriver; - } - - @Override - public void shutdown(ExecutorDriver executorDriver) { - log.debug("shutting down executor"); - shutdownPlugin(); - suicide(null, 0); - } - - - private List updateYamlFiles(List paths, ExecutorInfo executorInfo,TaskInfo taskInfo) throws IOException { - List newPath = new ArrayList<>(); - for (String file: paths) { - Map>> rootYaml = readFromFile(file); - System.out.println(" rootYaml dump: " + rootYaml.toString()); - Map>> updatedYaml = composeRewriteHelper.updateYaml(rootYaml, taskInfo, executorInfo); - System.out.println(" updatedYaml dump: "+updatedYaml.toString()); - String outputFileName = getOutputFileName(file); - System.out.println("outputFileName: "+outputFileName); - writeToFile(outputFileName, updatedYaml); - newPath.add(outputFileName); - } - System.out.println("updated Paths: "+newPath.toString()); - return newPath; - } - - private String getOutputFileName(String path){ - if(path != null ){ - StringBuffer buf = new StringBuffer(path.trim()); - buf.append(GENERATED_YAML_FILE_NAME); - return buf.toString(); - } - return null; - } - - private File writeToFile(String fileName,Map>> updatedRootYaml) throws IOException,FileNotFoundException { - File file = new File(fileName); - FileWriter fileWriter = new FileWriter(file); - DumperOptions options=new DumperOptions(); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - Yaml yaml = new Yaml(options); - yaml.dump(updatedRootYaml,fileWriter); - fileWriter.flush(); - fileWriter.close(); - return file; - } - - private Map>> readFromFile(String path) throws FileNotFoundException,IOException{ - FileReader fileReader = new FileReader(new File(path)); - @SuppressWarnings("unchecked") - DumperOptions options=new DumperOptions(); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - Yaml yaml = new Yaml(options); - Map>> yamlMap = (Map>>)yaml.load(fileReader); - fileReader.close(); - return yamlMap; - } +public class DockerComposeExecutor implements Executor { + + private static final Logger log = Logger.getLogger(DockerComposeExecutor.class); + private static final String GENERATED_YAML_FILE_NAME = "-generated.yml"; + + + private ComposeFileList fileFetcher; + + private List fileNames; + + private ExecutorDriver executorDriver; + + private DockerComposeProcessObserver processObserver; + + private ExecutorInfo executorInfo; + + private ComposeMonitor podMonitor; + + private ComposeRewriteHelper composeRewriteHelper; + + final PluginManager pluginManager = new DefaultPluginManager(); + + private boolean pluginEnabled = false; + + @Inject + public DockerComposeExecutor(ComposeFileList fileFetcher, DockerComposeProcessObserver processObserver, + ComposeMonitor podMonitor, ComposeRewriteHelper helper) { + this.fileFetcher = fileFetcher; + this.processObserver = processObserver; + this.podMonitor = podMonitor; + this.composeRewriteHelper = helper; + } + + @Override + public void launchTask(ExecutorDriver executorDriver, TaskInfo taskInfo) { + if (log.isDebugEnabled()) { + log.debug(" ############ launchTask #############"); + log.debug(" taskInfo: " + taskInfo.toString()); + log.debug(" executorDriver: " + executorDriver.toString()); + } + TaskID taskId = taskInfo.getTaskId(); + processObserver.init(this, taskId); + sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_STARTING); + try { + this.fileNames = executePlugin(executorDriver, taskInfo); + if (this.fileNames == null || this.fileNames.size() == 0) { + sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FAILED); + return; + } + + this.fileNames = updateYamlFiles(this.fileNames, executorInfo, taskInfo); + log.debug(" fileNames: " + this.fileNames.toString()); + podMonitor.subscribeToChanges(new Action1() { + @Override + public void call(Integer exitCode) { + suicide(taskId, exitCode); + } + }); + podMonitor.startMonitoring(this.fileNames); + updateImagesAndStartCompose(taskId); + sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_RUNNING); + } catch (Exception e) { + log.error("exception while launching process", e); + sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FAILED); + System.exit(1); + } + } + + private List executePlugin(ExecutorDriver executorDriver, TaskInfo taskInfo) { + if (this.pluginManager != null) { + List plugins = this.pluginManager.getExtensions(ComposeExecutorPlugin.class); + log.debug(String.format("Found %d extensions for extension point '%s'", plugins.size(), ComposeExecutorPlugin.class.getName())); + //check to ensure that only a single plugin exists.. + if (plugins != null && plugins.size() > 1) { + log.error(" more than one extension implementation: " + plugins.toString()); + return null; + } + return plugins.get(0).launchTask(executorDriver, taskInfo); + } else { + log.error(" pluginManager NULL "); + } + return null; + } + + private void shutdownPlugin() { + if (this.pluginManager != null) { + List plugins = this.pluginManager.getExtensions(ComposeExecutorPlugin.class); + log.debug(String.format("Found %d extensions for extension point '%s'", plugins.size(), ComposeExecutorPlugin.class.getName())); + if (plugins != null && plugins.size() > 1) { + log.error("More than one extension implementation: " + plugins.toString()); + return ; + } + plugins.get(0).shutdown(); + } + } + + private void updateImagesAndStartCompose(TaskID taskId) { + Observable.create(new Observable.OnSubscribe() { + @Override + public void call(Subscriber subscriber) { + String pullCommand = CommandBuilder.pullImages(fileNames); + log.debug(" pullCommand: " + pullCommand); + int imageUpdateExitCode = ProcessUtils.executeCommand(pullCommand, null); + if (imageUpdateExitCode != 0) { + log.error("unable to pull updated images trying to bring the pod up with existing images"); + } + String launchCommand = CommandBuilder.launchTask(fileNames); + log.debug(" launchCommand: " + launchCommand); + int exitCode = ProcessUtils.executeCommand(launchCommand, null); + subscriber.onNext(exitCode); + subscriber.onCompleted(); + } + }).subscribeOn(Schedulers.newThread()).observeOn(Schedulers.newThread()).subscribe(processObserver); + } + + public void suicide(TaskID taskId, int exitCode) { + + if (log.isDebugEnabled()) { + log.debug(" ############## suicide #######"); + log.debug("taskId: " + taskId.toString() + " exitCode: " + exitCode); + } + int stopContainers = cleanUp(); + + if (this.pluginManager != null) { + this.pluginManager.stopPlugins(); + } + + if (exitCode == 0 && stopContainers == 0) { + sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FINISHED); + System.exit(0); + } else { + sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FAILED); + System.exit(1); + } + } + + + /** + * stop docker-compose + * force remove docker images + * + * @return killing compose and removing all images are successful + */ + private int cleanUp() { + if (log.isDebugEnabled()) { + log.debug(" ############## cleanUp ###########, fileName: " + fileNames); + } + String killTask = CommandBuilder.stopTask(fileNames); + log.debug(" killTask: " + killTask); + int exitCode = ProcessUtils.executeCommand(killTask, null); + if (exitCode != 0) { + exitCode = linuxKill(fileNames); + } + return exitCode; + } + + private int linuxKill(List files) { + log.debug("########### linuxKill ##########, fileName: " + files); + List containerIds = podMonitor.getContainerIds(files); + List pids = new ArrayList(); + for (String containerId : containerIds) { + log.debug(" containerId: " + containerId); + ContainerDetails details = podMonitor.getContainerDetails(containerId); + log.debug(" details: " + details); + int pid = details.getPid(); + if (pid != 0) { + pids.add(pid); + } + } + int exitCode = 1; + if (pids.size() > 0) { + log.debug("pids: " + pids.toString()); + String command = CommandBuilder.linuxKill(pids); + log.debug(" command: " + command); + exitCode = ProcessUtils.executeCommand(command, null); + } + return exitCode; + } + + private void sendTaskStatusUpdate(ExecutorDriver executorDriver, TaskID taskId, TaskState taskState) { + TaskStatus taskStatus = TaskStatus.newBuilder().setTaskId(taskId).setState(taskState).build(); + executorDriver.sendStatusUpdate(taskStatus); + } + + @Override + public void killTask(ExecutorDriver executorDriver, TaskID taskId) { + log.info("killTask, taskId:" + taskId.getValue()); + suicide(taskId, 0); + } + + @Override + public void disconnected(ExecutorDriver executorDriver) { + log.debug("executor disconnected"); + } + + @Override + public void error(ExecutorDriver executorDriver, String errorMessage) { + log.error("executor received an error message:" + errorMessage); + } + + @Override + public void frameworkMessage(ExecutorDriver arg0, byte[] arg1) { + log.debug("received framework message"); + } + + @Override + public void registered(ExecutorDriver executorDriver, ExecutorInfo executorInfo, + FrameworkInfo frameworkInfo, SlaveInfo slaveInfo) { + log.debug("executor registered with framework:" + frameworkInfo.getName() + ":on slave:" + slaveInfo.getHostname()); + this.executorInfo = executorInfo; + this.executorDriver = executorDriver; + } + + @Override + public void reregistered(ExecutorDriver executorDriver, SlaveInfo slaveInfo) { + log.debug("executor reregistered on slave:" + slaveInfo.getHostname()); + this.executorDriver = executorDriver; + } + + @Override + public void shutdown(ExecutorDriver executorDriver) { + log.debug("shutting down executor"); + shutdownPlugin(); + suicide(null, 0); + } + + + public List updateYamlFiles(List paths, ExecutorInfo executorInfo, TaskInfo taskInfo) throws IOException { + List newPath = new ArrayList<>(); + for (String file : paths) { + Map>> rootYaml = readFromFile(file); + if (log.isDebugEnabled()) + log.debug(" rootYaml dump: " + rootYaml.toString()); + Map>> updatedYaml = composeRewriteHelper.updateYaml(rootYaml, taskInfo, executorInfo); + if (log.isDebugEnabled()) + log.debug(" updatedYaml dump: " + updatedYaml.toString()); + String outputFileName = getOutputFileName(file); + log.debug("outputFileName: " + outputFileName); + writeToFile(outputFileName, updatedYaml); + newPath.add(outputFileName); + } + log.debug("updated Paths: " + newPath.toString()); + return newPath; + } + + private String getOutputFileName(String path) { + if (path != null) { + StringBuffer buf = new StringBuffer(path.trim()); + buf.append(GENERATED_YAML_FILE_NAME); + return buf.toString(); + } + return null; + } + + private File writeToFile(String fileName, Map>> updatedRootYaml) throws IOException, FileNotFoundException { + File file = new File(fileName); + FileWriter fileWriter = new FileWriter(file); + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yaml = new Yaml(options); + yaml.dump(updatedRootYaml, fileWriter); + fileWriter.flush(); + fileWriter.close(); + return file; + } + + private Map>> readFromFile(String path) throws FileNotFoundException, IOException { + FileReader fileReader = new FileReader(new File(path)); + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yaml = new Yaml(options); + @SuppressWarnings("unchecked") + Map>> yamlMap = (Map>>) yaml.load(fileReader); + fileReader.close(); + return yamlMap; + } } diff --git a/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java b/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java index 723ccee..84ed468 100644 --- a/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java +++ b/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java @@ -5,37 +5,37 @@ import rx.Observer; -public class DockerComposeProcessObserver implements Observer{ +public class DockerComposeProcessObserver implements Observer { + + private static final Logger log = Logger.getLogger(DockerComposeProcessObserver.class); + + private DockerComposeExecutor executor; + private TaskID taskId; - private static final Logger log = Logger.getLogger(DockerComposeProcessObserver.class); - - private DockerComposeExecutor executor; - private TaskID taskId; - public DockerComposeProcessObserver() { - + } - - public void init(DockerComposeExecutor executor,TaskID taskId){ - this.executor = executor; - this.taskId = taskId; + + public void init(DockerComposeExecutor executor, TaskID taskId) { + this.executor = executor; + this.taskId = taskId; + } + + @Override + public void onCompleted() { + + } + + @Override + public void onError(Throwable e) { + log.error("failed to pull images or bring up docker-compose executor for taskId:" + taskId.getValue(), e); + executor.suicide(taskId, 1); + } + + @Override + public void onNext(Integer t) { + log.info("executor for taskId:" + taskId.getValue() + " exited with exitCode:" + t); + executor.suicide(taskId, t); } - - @Override - public void onCompleted() { - - } - - @Override - public void onError(Throwable e) { - log.error("failed to pull images or bring up docker-compose executor for taskId:"+taskId.getValue(),e); - executor.suicide(taskId, 1); - } - - @Override - public void onNext(Integer t) { - log.info("executor for taskId:"+taskId.getValue()+" exited with exitCode:"+t); - executor.suicide(taskId, t); - } } diff --git a/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java b/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java index 37772d2..6d9b65e 100644 --- a/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java +++ b/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java @@ -7,9 +7,9 @@ import dagger.Component; @Singleton -@Component(modules = { ComposeFileListModule.class,ExecutorModule.class}) +@Component(modules = {ComposeFileListModule.class, ExecutorModule.class}) public interface ExecutorComponent { - Executor getExecutor(); - + Executor getExecutor(); + } diff --git a/src/main/java/com/paypal/mesos/executor/ExecutorModule.java b/src/main/java/com/paypal/mesos/executor/ExecutorModule.java index ccda793..e4fb473 100644 --- a/src/main/java/com/paypal/mesos/executor/ExecutorModule.java +++ b/src/main/java/com/paypal/mesos/executor/ExecutorModule.java @@ -16,28 +16,36 @@ @Module public class ExecutorModule { - @Provides Executor provideDockerComposeExecutor(ComposeFileList fileFetcher,DockerComposeProcessObserver processObserver, - ComposeMonitor composeMonitor, ComposeRewriteHelper helper){ - return new DockerComposeExecutor(fileFetcher,processObserver,composeMonitor, helper); - } - - @Provides @Singleton ComposeMonitor provideComposeMonitor(){ - return new ComposeMonitor(); - } - - @Provides @Singleton DockerComposeProcessObserver provideDockerComposeProcessObserver(){ - return new DockerComposeProcessObserver(); - } - - @Provides @Singleton - ComposeRewriteHelper provideComposeRewriteHelper() { - return new ComposeRewriteHelper(); - } - - @Provides @Singleton Yaml provideYaml(){ - DumperOptions options=new DumperOptions(); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - return new Yaml(options); - } + @Provides + Executor provideDockerComposeExecutor(ComposeFileList fileFetcher, DockerComposeProcessObserver processObserver, + ComposeMonitor composeMonitor, ComposeRewriteHelper helper) { + return new DockerComposeExecutor(fileFetcher, processObserver, composeMonitor, helper); + } + + @Provides + @Singleton + ComposeMonitor provideComposeMonitor() { + return new ComposeMonitor(); + } + + @Provides + @Singleton + DockerComposeProcessObserver provideDockerComposeProcessObserver() { + return new DockerComposeProcessObserver(); + } + + @Provides + @Singleton + ComposeRewriteHelper provideComposeRewriteHelper() { + return new ComposeRewriteHelper(); + } + + @Provides + @Singleton + Yaml provideYaml() { + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + return new Yaml(options); + } } diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java index f1f3971..d125b47 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java @@ -7,6 +7,6 @@ public interface ComposeFileList { - List getFile(TaskInfo taskInfo) throws IOException; - + List getFile(TaskInfo taskInfo) throws IOException; + } diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java index cf9894e..5a1a290 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java @@ -14,121 +14,64 @@ import org.apache.mesos.Protos.TaskInfo; -public class ComposeFileListImpl implements ComposeFileList{ - - private static final Logger log = Logger.getLogger(ComposeFileListImpl.class); - -// private static final String GENERATED_YAML_FILE_NAME = "-generated.yml"; - private static final String FILE_DELIMITER = ","; -// -// private ComposeRewriteHelper helper; -// -// private Yaml yaml; - -// ExecutorInfo info; - -// public ComposeFileListImpl(DockerRewriteHelper helper,Yaml yaml) { -// this.helper = helper; -// this.yaml = yaml; -// } - - @Inject - public ComposeFileListImpl() { - - } - - @Override - public List getFile(TaskInfo taskInfo) throws FileNotFoundException,IOException{ - System.out.println("############## STARTING DockerComposeFileFetcher.getFile ##############"); - - List paths = getFileName(taskInfo); - System.out.println(" paths: " + paths); - validateFiles(paths); - - return paths; - // Refactored code .. move yaml update logic in a method.. -// return updateYamlFiles(paths, executorInfo, taskInfo); - } - -// private List updateYamlFiles(List paths, ExecutorInfo executorInfo,TaskInfo taskInfo) throws IOException { -// List newPath = new ArrayList<>(); -// for (String file: paths) { -// Map>> rootYaml = readFromFile(file); -// System.out.println(" rootYaml dump: " + rootYaml.toString()); -// Map>> updatedYaml = helper.updateYaml(rootYaml, taskInfo, executorInfo); -// System.out.println(" updatedYaml dump: "+updatedYaml.toString()); -// String outputFileName = getOutputFileName(file); -// System.out.println("outputFileName: "+outputFileName); -// writeToFile(outputFileName, updatedYaml); -// newPath.add(outputFileName); -// } -// return newPath; -// } - -/* private String getOutputFileName(String path){ - if(path != null ){ - StringBuffer buf = new StringBuffer(path.trim()); - buf.append(GENERATED_YAML_FILE_NAME); - return buf.toString(); - } - return null; - }*/ - - //TODO figure out a way to lookup a map instead of iteration - private List getFileName(TaskInfo taskInfo){ - - Labels labels = taskInfo.getLabels(); - for(Label label:labels.getLabelsList()){ - if("fileName".equals(label.getKey())){ - return Arrays.asList(label.getValue().split(FILE_DELIMITER)); - } - } - log.warn("error reading fileName from taskInfo"); - return null; - } - -/* - private File writeToFile(String fileName,Map>> updatedRootYaml) throws IOException,FileNotFoundException{ - File file = new File(fileName); - FileWriter fileWriter = new FileWriter(file); - yaml.dump(updatedRootYaml,fileWriter); - fileWriter.flush(); - fileWriter.close(); - return file; - } - - private Map>> readFromFile(String path) throws FileNotFoundException,IOException{ - FileReader fileReader = new FileReader(new File(path)); - @SuppressWarnings("unchecked") - Map>> yamlMap = (Map>>)yaml.load(fileReader); - fileReader.close(); - return yamlMap; - } -*/ - - - private void validateFiles(List paths) throws FileNotFoundException{ - System.out.println(" ########### validateFiles ###########"); - System.out.println(" paths: "+paths.toString()); - if(paths == null || paths.size() == 0 ){ - throw new FileNotFoundException("empty .yml/.yaml file list @"); - } - for (String name: paths) { - File file = new File(name); - if (!file.isFile() || !(name.endsWith(".yml") || name.endsWith(".yaml"))){ - throw new FileNotFoundException("No .yml/.yaml file found @"+name); - } - } - return; - } - - private boolean fileExists(List paths){ - for (String name: paths) { - File file = new File(name); - String fileName = file.getName(); - return file.isFile() && (fileName.endsWith(".yml") || fileName.endsWith(".yaml")); - } - return true; - } +public class ComposeFileListImpl implements ComposeFileList { + + private static final Logger log = Logger.getLogger(ComposeFileListImpl.class); + private static final String FILE_DELIMITER = ","; + + + @Inject + public ComposeFileListImpl() { + } + + @Override + public List getFile(TaskInfo taskInfo) throws FileNotFoundException, IOException { + + List paths = getFileName(taskInfo); + if (log.isDebugEnabled()) + log.debug("############## ComposeFileList.getFile, paths: ##############" + paths); + validateFiles(paths); + + return paths; + } + + + //TODO figure out a way to lookup a map instead of iteration + private List getFileName(TaskInfo taskInfo) { + + Labels labels = taskInfo.getLabels(); + for (Label label : labels.getLabelsList()) { + if ("fileName".equals(label.getKey())) { + return Arrays.asList(label.getValue().split(FILE_DELIMITER)); + } + } + log.warn("error reading fileName from taskInfo"); + return null; + } + + + private void validateFiles(List paths) throws FileNotFoundException { + if (paths == null || paths.size() == 0) { + log.error("empty .yml/.yaml file list @"); + throw new FileNotFoundException("empty .yml/.yaml file list @"); + } + for (String name : paths) { + File file = new File(name); + if (!file.isFile() || !(name.endsWith(".yml") || name.endsWith(".yaml"))) { + log.error("No .yml/.yaml file found @" + name); + throw new FileNotFoundException("No .yml/.yaml file found @" + name); + } + } + return; + } + + private boolean fileExists(List paths) { + for (String name : paths) { + File file = new File(name); + String fileName = file.getName(); + return file.isFile() && (fileName.endsWith(".yml") || fileName.endsWith(".yaml")); + } + return true; + } } diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java index a9ce197..a08d639 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java @@ -1,13 +1,8 @@ package com.paypal.mesos.executor.compose; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.IllegalFormatFlagsException; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; +import java.util.*; +import org.apache.log4j.Logger; import org.apache.mesos.Protos; import org.apache.mesos.Protos.ExecutorInfo; import org.apache.mesos.Protos.Resource; @@ -17,179 +12,183 @@ public class ComposeRewriteHelper { - - private static final String CONTAINER_NAME = "container_name"; - private static final String NETWORK = "network_mode"; - private static final String LINKS = "links"; - private static final String DEPENDS_ON = "depends_on"; - private static final String VOLUMES_FROM = "volumes_from"; - private static final String PORTS = "ports"; - private static final String LABELS = "labels"; - private static final String SERVICES = "services"; - - public Map>> updateYaml(Map>> yamlMap, - TaskInfo taskInfo, - ExecutorInfo executorInfo){ - - System.out.println(" ############ STARTING updateYaml ################"); - if(yamlMap == null || yamlMap.isEmpty()){ - return null; - } - Map>> resultantContainerMap = new HashMap>>(); - resultantContainerMap.putAll(yamlMap); - String taskId = taskInfo.getTaskId().getValue(); - System.out.println(" taskId: " + taskId); - - Iterator portIterator = getPortMappingIterator(taskInfo); - // System.out.println(" portIterator: " + portIterator.toString());// this doesnt help - String executorId = executorInfo.getExecutorId().getValue(); - System.out.println(" executorId: " + executorId); - Map> services = yamlMap.get(SERVICES); - System.out.println(" services map: " + services.toString()); - Map> resultantServicesMap = new HashMap>(); - for(Map.Entry> containerEntry:services.entrySet()){ - - String key = containerEntry.getKey(); - Map containerValue = containerEntry.getValue(); - Map updatedContainerValues = updateContainerValue(executorId, - taskInfo, - containerValue, - portIterator); - String updatedKey = prefixTaskId(taskId, key); - resultantServicesMap.put(updatedKey,updatedContainerValues); - } - resultantContainerMap.put(SERVICES, resultantServicesMap); - return resultantContainerMap; - } - - - private Map updateContainerValue(String executorId, - TaskInfo taskInfo, - Map containerDetails, - Iterator portIterator){ - - String taskId = taskInfo.getTaskId().getValue(); - System.out.println(" ################## Starting updateContainerValue ############### "); - System.out.println(" executorId: " + executorId + " taskId: " + taskId + " containerDetails: " + containerDetails.toString() + " portIterator: " + portIterator.toString()); - - if(containerDetails.containsKey(CONTAINER_NAME)){ - String containerValue = prefixTaskId(taskId,String.valueOf(containerDetails.get(CONTAINER_NAME))); - System.out.println("updated ContainerName: "+containerValue); - containerDetails.put(CONTAINER_NAME, containerValue); - } - - Object networkValue = containerDetails.get(NETWORK); - if(networkValue !=null && (String.valueOf(networkValue).contains("service"))){ - String networkValueString = String.valueOf(networkValue); - String [] split = networkValueString.split(":"); - String containerName = split[split.length-1]; - System.out.println("updated network: " + "service:" + prefixTaskId(taskId, containerName)); - containerDetails.put(NETWORK, "service:"+prefixTaskId(taskId, containerName)); - } - System.out.println("In update container values"); - Object linkValues = containerDetails.get(LINKS); - if(linkValues != null){ - System.out.println("links present"); - List updatedLinks = new ArrayList(); - @SuppressWarnings("unchecked") - List links = (ArrayList)linkValues; - for(String link:links){ - updatedLinks.add(prefixTaskId(taskId, link)+":"+link); - System.out.println(prefixTaskId(taskId, link)); - } - System.out.println(" updatedLinks: "+updatedLinks); - containerDetails.put(LINKS, updatedLinks); - } - - Object dependson = containerDetails.get(DEPENDS_ON); - if(dependson != null){ - List updatedLinks = new ArrayList(); - @SuppressWarnings("unchecked") - List links = (ArrayList)dependson; - for(String link:links){ - updatedLinks.add(prefixTaskId(taskId, link)); - } - System.out.println("updated Links: "+updatedLinks); - containerDetails.put(DEPENDS_ON, updatedLinks); - } - - Object volumesFromValues = containerDetails.get(VOLUMES_FROM); - if(volumesFromValues != null){ - List updatedVolumesFrom = new ArrayList(); - @SuppressWarnings("unchecked") - List volumesFrom = (ArrayList)volumesFromValues; - for(String volumeFrom:volumesFrom){ - updatedVolumesFrom.add(prefixTaskId(taskId, volumeFrom)); - } - System.out.println(" updated Volumes: "+updatedVolumesFrom); - containerDetails.put(VOLUMES_FROM, updatedVolumesFrom); - } - - Object portMappings = containerDetails.get(PORTS); - if(portMappings != null){ - List updatedPorts = new ArrayList(); - @SuppressWarnings("unchecked") - List portStrings = (ArrayList)portMappings; - for(String portString:portStrings){ - String replacedPort = replacePort(portString,portIterator); - updatedPorts.add(replacedPort); - } - System.out.println(" updatedPorts: "+updatedPorts); - containerDetails.put(PORTS, updatedPorts); - } - - @SuppressWarnings("unchecked") - Map taskIdLabel = (Map)containerDetails.get(LABELS); - if(taskIdLabel == null){ - taskIdLabel = new HashMap(); - } - taskIdLabel.put("taskId", taskId); - taskIdLabel.put("executorId",executorId); - - for(Protos.Label l : taskInfo.getLabels().getLabelsList()) { - taskIdLabel.put(l.getKey(), l.getValue()); - } - - System.out.println(" updated taskIdLabel: "+ taskIdLabel); - containerDetails.put(LABELS, taskIdLabel); - - return containerDetails; - } - - private String replacePort(String portString,Iterator portIterator){ - if(portIterator.hasNext()){ - String [] tokens = portString.split(":"); - if(tokens.length > 1){ - return portIterator.next()+":"+tokens[1]; - }else{ - throw new IllegalFormatFlagsException("port mappings in docker-compose file not valid"); - } - }else{ - throw new NoSuchElementException("Insufficient number of ports allocated"); - } - } - - private Iterator getPortMappingIterator(TaskInfo taskInfo){ - List list = taskInfo.getResourcesList(); - List ports = new ArrayList(); - for(Resource resource:list){ - String name = resource.getName(); - if("ports".equals(name)){ - Ranges ranges = resource.getRanges(); - for(Range range:ranges.getRangeList()){ - long startPort = range.getBegin(); - long endPort = range.getEnd(); - for(int i=0;i<=endPort-startPort;i++){ - ports.add(startPort+i); - } - } - } - } - return ports.iterator(); - } - - private String prefixTaskId(String taskId,String key){ - StringBuilder builder = new StringBuilder(); - return builder.append(taskId).append("_").append(key).toString(); - } + + private static final String CONTAINER_NAME = "container_name"; + private static final String NETWORK = "network_mode"; + private static final String LINKS = "links"; + private static final String DEPENDS_ON = "depends_on"; + private static final String VOLUMES_FROM = "volumes_from"; + private static final String PORTS = "ports"; + private static final String LABELS = "labels"; + private static final String SERVICES = "services"; + + private static final Logger log = Logger.getLogger(ComposeRewriteHelper.class); + + public Map>> updateYaml(Map>> yamlMap, + TaskInfo taskInfo, + ExecutorInfo executorInfo) { + + log.debug(" ############ STARTING updateYaml ################"); + if (yamlMap == null || yamlMap.isEmpty()) { + return null; + } + Map>> resultantContainerMap = new HashMap>>(); + resultantContainerMap.putAll(yamlMap); + String taskId = taskInfo.getTaskId().getValue(); + + Iterator portIterator = getPortMappingIterator(taskInfo); + String executorId = executorInfo.getExecutorId().getValue(); + log.debug(" executorId: " + executorId); + Map> services = yamlMap.get(SERVICES); + log.debug(" services map: " + services.toString()); + Map> resultantServicesMap = new HashMap>(); + for (Map.Entry> containerEntry : services.entrySet()) { + + String key = containerEntry.getKey(); + Map containerValue = containerEntry.getValue(); + Map updatedContainerValues = updateContainerValue(executorId, + taskInfo, + containerValue, + portIterator); + String updatedKey = prefixTaskId(taskId, key); + resultantServicesMap.put(updatedKey, updatedContainerValues); + } + resultantContainerMap.put(SERVICES, resultantServicesMap); + return resultantContainerMap; + } + + + private Map updateContainerValue(String executorId, + TaskInfo taskInfo, + Map containerDetails, + Iterator portIterator) { + + String taskId = taskInfo.getTaskId().getValue(); + + if (log.isDebugEnabled()) { + log.debug(" ################## Starting updateContainerValue ############### "); + log.debug(" executorId: " + executorId + " taskId: " + taskId + " containerDetails: " + containerDetails.toString() + " portIterator: " + portIterator.toString()); + } + + if (containerDetails.containsKey(CONTAINER_NAME)) { + String containerValue = prefixTaskId(taskId, String.valueOf(containerDetails.get(CONTAINER_NAME))); + log.debug("updated ContainerName: " + containerValue); + containerDetails.put(CONTAINER_NAME, containerValue); + } + + Object networkValue = containerDetails.get(NETWORK); + if (networkValue != null && (String.valueOf(networkValue).contains("service"))) { + String networkValueString = String.valueOf(networkValue); + String[] split = networkValueString.split(":"); + String containerName = split[split.length - 1]; + log.debug("updated network: " + "service:" + prefixTaskId(taskId, containerName)); + containerDetails.put(NETWORK, "service:" + prefixTaskId(taskId, containerName)); + } + + Object linkValues = containerDetails.get(LINKS); + if (linkValues != null) { + log.debug("links present"); + List updatedLinks = new ArrayList(); + List links = (List) linkValues; + for (Object iter : links) { + String link = (String) iter; + updatedLinks.add(prefixTaskId(taskId, link) + ":" + link); + log.debug(prefixTaskId(taskId, link)+ ":" + link); + } + log.debug(" updatedLinks: " + updatedLinks); + containerDetails.put(LINKS, updatedLinks); + } + + Object dependson = containerDetails.get(DEPENDS_ON); + if (dependson != null) { + List updatedDependsOn = new ArrayList(); + List dependsOnValues = (List) dependson; + for (Object o : dependsOnValues) { + String dependsOn = (String) o; + updatedDependsOn.add(prefixTaskId(taskId, dependsOn)); + } + log.debug("updated DependsOn: " + updatedDependsOn); + containerDetails.put(DEPENDS_ON, updatedDependsOn); + } + + Object volumesFromValues = containerDetails.get(VOLUMES_FROM); + if (volumesFromValues != null) { + List updatedVolumesFrom = new ArrayList(); + List volumesFrom = (List) updatedVolumesFrom; + + for (Object o : volumesFrom) { + String volume = (String) o; + updatedVolumesFrom.add(prefixTaskId(taskId, volume)); + } + log.debug(" updated Volumes: " + updatedVolumesFrom); + containerDetails.put(VOLUMES_FROM, updatedVolumesFrom); + } + + Object portMappings = containerDetails.get(PORTS); + if (portMappings != null) { + List updatedPorts = new ArrayList(); + List ports = (List) portMappings; + for (Object o : ports) { + String port = (String) o; + String replacedPort = replacePort(port, portIterator); + updatedPorts.add(replacedPort); + } + log.debug(" updatedPorts: " + updatedPorts); + containerDetails.put(PORTS, updatedPorts); + } + @SuppressWarnings("unchecked") + Map taskIdLabel = (Map) containerDetails.get(LABELS); + if (taskIdLabel == null) { + taskIdLabel = new HashMap(); + } + taskIdLabel.put("taskId", taskId); + taskIdLabel.put("executorId", executorId); + for (Protos.Label l : taskInfo.getLabels().getLabelsList()) { + taskIdLabel.put(l.getKey(), l.getValue()); + } + + log.debug(" updated taskIdLabel: " + taskIdLabel); + containerDetails.put(LABELS, taskIdLabel); + + return containerDetails; + } + + private String replacePort(String portString, Iterator portIterator) { + if (portIterator.hasNext()) { + String[] tokens = portString.split(":"); + if (tokens.length > 1) { + return portIterator.next() + ":" + tokens[1]; + } else { + throw new IllegalFormatFlagsException("port mappings in docker-compose file not valid"); + } + } else { + throw new NoSuchElementException("Insufficient number of ports allocated"); + } + } + + private Iterator getPortMappingIterator(TaskInfo taskInfo) { + List list = taskInfo.getResourcesList(); + List ports = new ArrayList(); + for (Resource resource : list) { + String name = resource.getName(); + if ("ports".equals(name)) { + Ranges ranges = resource.getRanges(); + for (Range range : ranges.getRangeList()) { + long startPort = range.getBegin(); + long endPort = range.getEnd(); + for (int i = 0; i <= endPort - startPort; i++) { + ports.add(startPort + i); + } + } + } + } + return ports.iterator(); + } + + private String prefixTaskId(String taskId, String key) { + StringBuilder builder = new StringBuilder(); + return builder.append(taskId).append("_").append(key).toString(); + } + + } diff --git a/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java b/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java index 0debd69..a931eeb 100644 --- a/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java +++ b/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java @@ -22,107 +22,108 @@ //TODO remode public class ComposeMonitor { - private static final Logger log = Logger.getLogger(ComposeMonitor.class); - - public final BehaviorSubject monitor; - - public ComposeMonitor(){ - monitor = BehaviorSubject.create(); - } - - public Subscription subscribeToChanges(Action1 action){ - return monitor.subscribe(action); - } - - public void startMonitoring(final List fileNames){ - log.info("start montioring is called:"+fileNames); - Observable.interval(Config.POD_MONITOR_INTERVAL, TimeUnit.MILLISECONDS).subscribe(new Observer() { - - @Override - public void onCompleted() { - log.info("monitor thread on completed :completed monitoring compose for file:"+fileNames); - } - - @Override - public void onError(Throwable e) { - log.error("monitor thread on error: encountred an error monitoring:"+fileNames,e); - monitor.onNext(1); - } - - @Override - public void onNext(Long t) { - List containerIds = getContainerIds(fileNames); - System.out.println(Arrays.toString(containerIds.toArray())); - if(containerIds != null){ - for(String containerId:containerIds){ - ContainerDetails details = getContainerDetails(containerId); - int exitCode = details.getExitCode(); - int restartCount = details.getRestartCount(); - if(details.isRunning() == false && exitCode != 0 && restartCount == details.getMaxAllowedRestartCount()){ - RestartPolicyException exception = new RestartPolicyException(containerId, exitCode, restartCount); - onError(exception); - } - } - } - } - }); - - } - - public List getContainerIds(List fileNames){ - List containerIds = new ArrayList(); - String listConatinerIdsCommand = CommandBuilder.getContainerIds(fileNames); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - int exitCode = ProcessUtils.executeCommand(listConatinerIdsCommand, null, outputStream, null, null); - if(exitCode == 0){ - String commandOutput = outputStream.toString(); - containerIds = parseListCommandOutput(commandOutput); - } - return containerIds; - } - - private List parseListCommandOutput(String output){ - List containersIds = new ArrayList(); - if(output != null){ - StringTokenizer tokenizer = new StringTokenizer(output,"\n"); - while(tokenizer.hasMoreTokens()){ - containersIds.add(tokenizer.nextToken().trim()); - } - } - return containersIds; - } - - public ContainerDetails getContainerDetails(String containerId){ - String detailsCommand = CommandBuilder.getContainerDetails(containerId); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - int exitCode = ProcessUtils.executeCommand(detailsCommand, null, outputStream, null, null); - if(exitCode == 0){ - String details = outputStream.toString(); - ContainerDetails containerDetails = parseDetails(containerId,details); - return containerDetails; - }else{ - throw new ContainerDetailRetrievalException(containerId); - } - } - - public ContainerDetails parseDetails(String containerId,String details){ - StringTokenizer tokenizer = new StringTokenizer(details,","); - if(tokenizer.countTokens() == 5){ - try{ - int pid = Integer.parseInt(tokenizer.nextToken()); - int exitCode = Integer.parseInt(tokenizer.nextToken()); - boolean isRunning = Boolean.parseBoolean(tokenizer.nextToken()); - int restartCount = Integer.parseInt(tokenizer.nextToken()); - int maxRestartCount = Integer.parseInt(tokenizer.nextToken().trim()); - return new ContainerDetails(containerId,isRunning, restartCount, maxRestartCount, exitCode,pid); - }catch(Exception exception){ - log.error("problem while parsing container details for containerId:"+containerId+":details string is:"+details, exception); - throw new ContainerDetailRetrievalException(containerId); - } - }else{ - throw new ContainerDetailRetrievalException(containerId); - } - } + private static final Logger log = Logger.getLogger(ComposeMonitor.class); + + public final BehaviorSubject monitor; + + public ComposeMonitor() { + monitor = BehaviorSubject.create(); + } + + public Subscription subscribeToChanges(Action1 action) { + return monitor.subscribe(action); + } + + public void startMonitoring(final List fileNames) { + log.info("start montioring is called:" + fileNames); + Observable.interval(Config.POD_MONITOR_INTERVAL, TimeUnit.MILLISECONDS).subscribe(new Observer() { + + @Override + public void onCompleted() { + log.info("monitor thread on completed :completed monitoring compose for file:" + fileNames); + } + + @Override + public void onError(Throwable e) { + log.error("monitor thread on error: encountred an error monitoring:" + fileNames, e); + monitor.onNext(1); + } + + @Override + public void onNext(Long t) { + List containerIds = getContainerIds(fileNames); + if (log.isDebugEnabled()) + log.debug(Arrays.toString(containerIds.toArray())); + if (containerIds != null) { + for (String containerId : containerIds) { + ContainerDetails details = getContainerDetails(containerId); + int exitCode = details.getExitCode(); + int restartCount = details.getRestartCount(); + if (details.isRunning() == false && exitCode != 0 && restartCount == details.getMaxAllowedRestartCount()) { + RestartPolicyException exception = new RestartPolicyException(containerId, exitCode, restartCount); + onError(exception); + } + } + } + } + }); + + } + + public List getContainerIds(List fileNames) { + List containerIds = new ArrayList(); + String listConatinerIdsCommand = CommandBuilder.getContainerIds(fileNames); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + int exitCode = ProcessUtils.executeCommand(listConatinerIdsCommand, null, outputStream, null, null); + if (exitCode == 0) { + String commandOutput = outputStream.toString(); + containerIds = parseListCommandOutput(commandOutput); + } + return containerIds; + } + + private List parseListCommandOutput(String output) { + List containersIds = new ArrayList(); + if (output != null) { + StringTokenizer tokenizer = new StringTokenizer(output, "\n"); + while (tokenizer.hasMoreTokens()) { + containersIds.add(tokenizer.nextToken().trim()); + } + } + return containersIds; + } + + public ContainerDetails getContainerDetails(String containerId) { + String detailsCommand = CommandBuilder.getContainerDetails(containerId); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + int exitCode = ProcessUtils.executeCommand(detailsCommand, null, outputStream, null, null); + if (exitCode == 0) { + String details = outputStream.toString(); + ContainerDetails containerDetails = parseDetails(containerId, details); + return containerDetails; + } else { + throw new ContainerDetailRetrievalException(containerId); + } + } + + public ContainerDetails parseDetails(String containerId, String details) { + StringTokenizer tokenizer = new StringTokenizer(details, ","); + if (tokenizer.countTokens() == 5) { + try { + int pid = Integer.parseInt(tokenizer.nextToken()); + int exitCode = Integer.parseInt(tokenizer.nextToken()); + boolean isRunning = Boolean.parseBoolean(tokenizer.nextToken()); + int restartCount = Integer.parseInt(tokenizer.nextToken()); + int maxRestartCount = Integer.parseInt(tokenizer.nextToken().trim()); + return new ContainerDetails(containerId, isRunning, restartCount, maxRestartCount, exitCode, pid); + } catch (Exception exception) { + log.error("problem while parsing container details for containerId:" + containerId + ":details string is:" + details, exception); + throw new ContainerDetailRetrievalException(containerId); + } + } else { + throw new ContainerDetailRetrievalException(containerId); + } + } } diff --git a/src/main/java/com/paypal/mesos/executor/monitoring/ContainerDetails.java b/src/main/java/com/paypal/mesos/executor/monitoring/ContainerDetails.java index ad870a6..4c0023e 100644 --- a/src/main/java/com/paypal/mesos/executor/monitoring/ContainerDetails.java +++ b/src/main/java/com/paypal/mesos/executor/monitoring/ContainerDetails.java @@ -2,57 +2,57 @@ public class ContainerDetails { - private String containerId; - - private boolean isRunning; - - private int restartCount; - - private int maxAllowedRestartCount; - - private int exitCode; - - private int pid; - - public ContainerDetails(String containerId,boolean isRunning,int restartCount,int maxAllowedRestartCount,int exitCode,int pid){ - this.containerId = containerId; - this.isRunning = isRunning; - this.restartCount = restartCount; - this.maxAllowedRestartCount = maxAllowedRestartCount; - this.exitCode = exitCode; - this.pid = pid; - } - - public String getContainerId() { - return containerId; - } - - - public boolean isRunning() { - return isRunning; - } - - public int getRestartCount() { - return restartCount; - } - - public int getMaxAllowedRestartCount() { - return maxAllowedRestartCount; - } - - public int getExitCode() { - return exitCode; - } - - public int getPid() { - return pid; - } - - @Override - public String toString() { - return "ContainerDetails [containerId=" + containerId + ", isRunning=" + isRunning + ", restartCount=" - + restartCount + ", maxAllowedRestartCount=" - + maxAllowedRestartCount + ", exitCode=" + exitCode + "]"; - } - + private String containerId; + + private boolean isRunning; + + private int restartCount; + + private int maxAllowedRestartCount; + + private int exitCode; + + private int pid; + + public ContainerDetails(String containerId, boolean isRunning, int restartCount, int maxAllowedRestartCount, int exitCode, int pid) { + this.containerId = containerId; + this.isRunning = isRunning; + this.restartCount = restartCount; + this.maxAllowedRestartCount = maxAllowedRestartCount; + this.exitCode = exitCode; + this.pid = pid; + } + + public String getContainerId() { + return containerId; + } + + + public boolean isRunning() { + return isRunning; + } + + public int getRestartCount() { + return restartCount; + } + + public int getMaxAllowedRestartCount() { + return maxAllowedRestartCount; + } + + public int getExitCode() { + return exitCode; + } + + public int getPid() { + return pid; + } + + @Override + public String toString() { + return "ContainerDetails [containerId=" + containerId + ", isRunning=" + isRunning + ", restartCount=" + + restartCount + ", maxAllowedRestartCount=" + + maxAllowedRestartCount + ", exitCode=" + exitCode + "]"; + } + } diff --git a/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java index a026d26..e41c532 100644 --- a/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java +++ b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java @@ -3,19 +3,15 @@ import org.apache.mesos.ExecutorDriver; import org.apache.mesos.Protos; import ro.fortsoft.pf4j.ExtensionPoint; - -import java.io.IOException; import java.util.List; /** - * Created by kkrishna on 7/18/16. + * Introduced to establish a hook at launchTask and shutdown executor callbacks. */ public interface ComposeExecutorPlugin extends ExtensionPoint { - public List launchTask(ExecutorDriver executorDriver, Protos.TaskInfo taskInfo); - - public void shutdown(); + public List launchTask(ExecutorDriver executorDriver, Protos.TaskInfo taskInfo); - public List getComposeFiles(); + public void shutdown(); } diff --git a/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPluginImpl.java b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPluginImpl.java index 537bcd5..8b797af 100644 --- a/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPluginImpl.java +++ b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPluginImpl.java @@ -2,6 +2,7 @@ import com.paypal.mesos.executor.compose.ComposeFileList; import com.paypal.mesos.executor.compose.ComposeFileListImpl; +import org.apache.log4j.Logger; import org.apache.mesos.ExecutorDriver; import org.apache.mesos.Protos; import ro.fortsoft.pf4j.Extension; @@ -9,43 +10,42 @@ import java.io.IOException; import java.util.List; -/** - * Created by kkrishna on 7/18/16. +/* + * Introduced to establish a hook at launchTask and shutdown executor callbacks. + * Default implementation for launchTask, retrieves yaml files from TaskInfo object. This method is to be over-ridden as needed on case by case basis. + * launchTask method should return list of yaml files path. */ @Extension public class ComposeExecutorPluginImpl implements ComposeExecutorPlugin { + + private static final Logger log = Logger.getLogger(ComposeExecutorPluginImpl.class); + @Override - public List launchTask(ExecutorDriver executorDriver, Protos.TaskInfo taskInfo) { - System.out.println(" ######### ComposeExecutorPlugin launchTask #############"); - System.out.println(" ExecutorDriver: " + executorDriver.toString()); - System.out.println(" taskInfo: "+ taskInfo.toString()); + public List launchTask(ExecutorDriver executorDriver, Protos.TaskInfo taskInfo) { + + if (log.isDebugEnabled()) { + log.debug(" ######### ComposeExecutorPlugin launchTask #############"); + log.debug(" ExecutorDriver: " + executorDriver.toString()); + log.debug(" taskInfo: " + taskInfo.toString()); + } + ComposeFileList composeFiles = new ComposeFileListImpl(); Protos.TaskID taskId = taskInfo.getTaskId(); try { - return composeFiles.getFile(taskInfo); + return composeFiles.getFile(taskInfo); } catch (IOException e) { e.printStackTrace(); Protos.TaskStatus taskStatus = Protos.TaskStatus.newBuilder().setTaskId(taskId).setState(Protos.TaskState.TASK_FAILED).build(); executorDriver.sendStatusUpdate(taskStatus); } -// Protos.Labels labels = taskInfo.getLabels(); -// System.out.println("taskInfo labels: " + labels.toString()); -// labels.toBuilder().addLabels(Protos.Label.newBuilder().setKey("testkey").setValue("testvalue")).build(); -// taskInfo.toBuilder().setLabels(labels).build(); -// -// System.out.println(" taskInfo: " + taskInfo.toString()); - return null; + return null; } @Override public void shutdown() { - System.out.println(" ############## ComposeExecutorPlugin Shutdown ############"); + log.debug(" ############## ComposeExecutorPlugin Shutdown ############"); } - @Override - public List getComposeFiles() { - return null; - } } diff --git a/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java b/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java index fd9b886..26a24cf 100644 --- a/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java +++ b/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java @@ -14,54 +14,51 @@ public class ProcessUtils { - private static Logger log = Logger.getLogger(ProcessUtils.class); - - public static int executeCommand(String command,ExecuteWatchdog watchdog) { - return executeCommand(command, watchdog,null,null,null); - } - - public static int executeCommand(String command,ExecuteWatchdog watchdog,OutputStream outputStream,OutputStream errorStream,InputStream inputStream){ - System.out.println(" command: "+command); - CommandLine cmdLine = CommandLine.parse(command); - DefaultExecutor executor = new DefaultExecutor(); - if(outputStream == null){ - outputStream = System.out; - } - if(errorStream == null){ - errorStream = System.err; - } - executor.setStreamHandler(new PumpStreamHandler(outputStream,errorStream, inputStream)); - executor.setExitValues(new int[]{0, 1}); - if(watchdog != null){ - executor.setWatchdog(watchdog); - } - int exitValue = 0; - try { - exitValue = executor.execute(cmdLine); - } catch (IOException e) { - exitValue = 1; - log.error("error executing command", e); - } + private static Logger log = Logger.getLogger(ProcessUtils.class); + + public static int executeCommand(String command, ExecuteWatchdog watchdog) { + return executeCommand(command, watchdog, null, null, null); + } + + public static int executeCommand(String command, ExecuteWatchdog watchdog, OutputStream outputStream, OutputStream errorStream, InputStream inputStream) { + CommandLine cmdLine = CommandLine.parse(command); + DefaultExecutor executor = new DefaultExecutor(); + if (outputStream == null) { + outputStream = System.out; + } + if (errorStream == null) { + errorStream = System.err; + } + executor.setStreamHandler(new PumpStreamHandler(outputStream, errorStream, inputStream)); + executor.setExitValues(new int[]{0, 1}); + if (watchdog != null) { + executor.setWatchdog(watchdog); + } + int exitValue = 0; + try { + exitValue = executor.execute(cmdLine); + } catch (IOException e) { + exitValue = 1; + log.error("error executing command", e); + } + + return exitValue; + } + + public static ExecuteWatchdog createTimeoutWatchdog(TimeUnit timeunit, int timeout) { + ExecuteWatchdog timeoutWatchdog = new ExecuteWatchdog(timeunit.toMillis(timeout)); + return timeoutWatchdog; + } + + public static boolean isProcessRunning(int pid) { + String line; + if (OS.isFamilyWindows()) { + line = "cmd /c \"tasklist /FI \"PID eq " + pid + "\" | findstr " + pid + "\""; + } else { + line = "ps -p " + pid; + } + int exitValue = ProcessUtils.executeCommand(line, null); + return exitValue == 0; + } - System.out.println(" output: "+outputStream.toString()); - return exitValue; - } - - public static ExecuteWatchdog createTimeoutWatchdog(TimeUnit timeunit,int timeout){ - ExecuteWatchdog timeoutWatchdog = new ExecuteWatchdog(timeunit.toMillis(timeout)); - return timeoutWatchdog; - } - - public static boolean isProcessRunning(int pid) { - String line; - if (OS.isFamilyWindows()) { - line = "cmd /c \"tasklist /FI \"PID eq " + pid + "\" | findstr " + pid + "\""; - } - else { - line = "ps -p " + pid; - } - int exitValue = ProcessUtils.executeCommand(line, null); - return exitValue == 0; - } - } diff --git a/src/test/java/com/paypal/mesos/executor/compose/ComposeRewriteHelperTest.java b/src/test/java/com/paypal/mesos/executor/compose/ComposeRewriteHelperTest.java new file mode 100644 index 0000000..4dce11d --- /dev/null +++ b/src/test/java/com/paypal/mesos/executor/compose/ComposeRewriteHelperTest.java @@ -0,0 +1,70 @@ +package com.paypal.mesos.executor.compose; + +import com.paypal.mesos.executor.DockerComposeExecutor; +import com.paypal.mesos.executor.DockerComposeProcessObserver; +import com.paypal.mesos.executor.monitoring.ComposeMonitor; +import junit.framework.Test; +import org.apache.mesos.Protos; +import com.paypal.mesos.executor.compose.ComposeRewriteHelper; +import org.junit.Assert; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; + +import java.io.*; +import java.util.*; + + +/** + * Created by kkrishna on 8/2/16. + */ +public class ComposeRewriteHelperTest { + + DockerComposeExecutor composeExecutor = new DockerComposeExecutor( new ComposeFileListImpl(), + new DockerComposeProcessObserver(), + new ComposeMonitor(), + new ComposeRewriteHelper() ); + + @org.junit.Test + public void testTransformYaml() { + + /* resources { + name: "ports" + type: RANGES + ranges { + range { + begin: + 31227 + end: + 31229 + } + } + role: + "*" + }*/ + Protos.TaskInfo info = Protos.TaskInfo.newBuilder() + .setTaskId(Protos.TaskID.newBuilder().setValue("test.b8b72bb1-5901-11e6-af2b-0242a9458e21")) + .setLabels(Protos.Labels.newBuilder().addLabels(Protos.Label.newBuilder().setKey("key1").setValue("value1"))) + .addResources(Protos.Resource.newBuilder().setName("ports").setType(Protos.Value.Type.valueOf(1)) + .setRanges(Protos.Value.Ranges.newBuilder().addRange(Protos.Value.Range.newBuilder().setBegin(31227L).setEnd(31230L)))) + .setSlaveId(Protos.SlaveID.newBuilder().setValue("slave-id-b8b72bb1-5901-11e6-af2b-0242a9458e21")) + .buildPartial(); + System.out.println("taskInfo: " + info.toString()); + + Protos.ExecutorInfo executorInfo = Protos.ExecutorInfo.newBuilder(). + setExecutorId(Protos.ExecutorID.newBuilder().setValue("test.b0b5d762-590e-11e6-af2b-0242a9458e21")) + .buildPartial(); + System.out.println("executorInfo: " + executorInfo.toString()); + List files = Arrays.asList("src/test/resources/docker-compose.yml"); + + try { + List updatedFiles = composeExecutor.updateYamlFiles(files, executorInfo, info); + Assert.assertEquals(files.size(), updatedFiles.size()); + System.out.println("updated files: " + updatedFiles.toString()); + } catch (IOException e) { + e.printStackTrace(); + } + + } + +} + diff --git a/src/test/resources/docker-compose.yml b/src/test/resources/docker-compose.yml new file mode 100644 index 0000000..f51202d --- /dev/null +++ b/src/test/resources/docker-compose.yml @@ -0,0 +1,12 @@ +version: '2' +services: + web: + build: . + ports: + - 5000:5000 + volumes: + - .:/code + links: + - redis + redis: + image: redis \ No newline at end of file From 452a23c861af7699bd0e3dc4395cbc59954f4737 Mon Sep 17 00:00:00 2001 From: Kumar Krishna Date: Wed, 3 Aug 2016 15:09:44 -0700 Subject: [PATCH 04/15] trim fileName labels --- .../paypal/mesos/executor/compose/ComposeFileListImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java index 5a1a290..1c65047 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java @@ -42,7 +42,10 @@ private List getFileName(TaskInfo taskInfo) { Labels labels = taskInfo.getLabels(); for (Label label : labels.getLabelsList()) { if ("fileName".equals(label.getKey())) { - return Arrays.asList(label.getValue().split(FILE_DELIMITER)); + List files = Arrays.asList(label.getValue().split(FILE_DELIMITER)); + for (int i = 0; i < files.size(); i++) + files.set(i, files.get(i).trim()); + return files; } } log.warn("error reading fileName from taskInfo"); From 85475881b5b53534b0d8a40f18a35ac60f84b412 Mon Sep 17 00:00:00 2001 From: Kumar Krishna Date: Thu, 4 Aug 2016 13:52:17 -0700 Subject: [PATCH 05/15] adhere renaming in compose to rfc --- .../mesos/executor/compose/ComposeRewriteHelper.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java index a08d639..6e54952 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java @@ -187,7 +187,13 @@ private Iterator getPortMappingIterator(TaskInfo taskInfo) { private String prefixTaskId(String taskId, String key) { StringBuilder builder = new StringBuilder(); - return builder.append(taskId).append("_").append(key).toString(); + builder.append(key.toString()).append("-").append(taskId); + String newId = builder.toString(); + + newId.replaceAll("[^a-zA-Z0-9-]" ,""); + if (newId.length() > 63) + newId = newId.substring(0,63); + return newId; } From 81913422af38114b845776917a851f53cd85f8f9 Mon Sep 17 00:00:00 2001 From: Kumar Krishna Date: Fri, 5 Aug 2016 13:46:13 -0700 Subject: [PATCH 06/15] disable sending TASK_FAILED to mesos during killTask --- .../mesos/executor/DockerComposeExecutor.java | 60 +++++++++++++++---- .../DockerComposeProcessObserver.java | 4 +- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java b/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java index 4b1ce76..fbbda43 100644 --- a/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java +++ b/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java @@ -56,6 +56,8 @@ public class DockerComposeExecutor implements Executor { final PluginManager pluginManager = new DefaultPluginManager(); + private volatile boolean isShutDownInProgress = false; + private boolean pluginEnabled = false; @Inject @@ -94,6 +96,7 @@ public void call(Integer exitCode) { }); podMonitor.startMonitoring(this.fileNames); updateImagesAndStartCompose(taskId); + log.debug("sending TASK_RUNNING status update"); sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_RUNNING); } catch (Exception e) { log.error("exception while launching process", e); @@ -143,6 +146,7 @@ public void call(Subscriber subscriber) { String launchCommand = CommandBuilder.launchTask(fileNames); log.debug(" launchCommand: " + launchCommand); int exitCode = ProcessUtils.executeCommand(launchCommand, null); + log.debug("updateImagesAndStartCompose exit code: "+exitCode); subscriber.onNext(exitCode); subscriber.onCompleted(); } @@ -153,21 +157,46 @@ public void suicide(TaskID taskId, int exitCode) { if (log.isDebugEnabled()) { log.debug(" ############## suicide #######"); - log.debug("taskId: " + taskId.toString() + " exitCode: " + exitCode); - } - int stopContainers = cleanUp(); - - if (this.pluginManager != null) { - this.pluginManager.stopPlugins(); + if (taskId != null) + log.debug("taskId: " + taskId.toString() + " exitCode: " + exitCode); } - if (exitCode == 0 && stopContainers == 0) { + if (this.isShutDownInProgress && exitCode == 0) { + // proceed with finish task + int stopContainers = cleanUp(); + if (this.pluginManager != null) { + this.pluginManager.stopPlugins(); + } + log.debug(" cleanUp exit code: " + stopContainers); sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FINISHED); System.exit(0); } else { - sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FAILED); - System.exit(1); + if (!isShutDownInProgress) { + int stopContainers = cleanUp(); + if (this.pluginManager != null) { + this.pluginManager.stopPlugins(); + } + sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FAILED); + System.exit(1); + } else { + log.debug(" shutdown already in progress.."); + } + } +// +// int stopContainers = cleanUp(); +// +// if (this.pluginManager != null) { +// this.pluginManager.stopPlugins(); +// } +// +// if (exitCode == 0 && stopContainers == 0) { +// sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FINISHED); +// System.exit(0); +// } else { +// sendTaskStatusUpdate(executorDriver, taskId, TaskState.TASK_FAILED); +// System.exit(1); +// } } @@ -184,6 +213,8 @@ private int cleanUp() { String killTask = CommandBuilder.stopTask(fileNames); log.debug(" killTask: " + killTask); int exitCode = ProcessUtils.executeCommand(killTask, null); + log.debug(" cleanUp killTask exitCode: " + exitCode); + if (exitCode != 0) { exitCode = linuxKill(fileNames); } @@ -210,17 +241,23 @@ private int linuxKill(List files) { log.debug(" command: " + command); exitCode = ProcessUtils.executeCommand(command, null); } + log.debug(" linuxKill exitCode: "+exitCode); return exitCode; } private void sendTaskStatusUpdate(ExecutorDriver executorDriver, TaskID taskId, TaskState taskState) { - TaskStatus taskStatus = TaskStatus.newBuilder().setTaskId(taskId).setState(taskState).build(); - executorDriver.sendStatusUpdate(taskStatus); + if (taskId != null) { + TaskStatus taskStatus = TaskStatus.newBuilder().setTaskId(taskId).setState(taskState).build(); + executorDriver.sendStatusUpdate(taskStatus); + } else { + log.error("taskId is null"); + } } @Override public void killTask(ExecutorDriver executorDriver, TaskID taskId) { log.info("killTask, taskId:" + taskId.getValue()); + this.isShutDownInProgress = true; suicide(taskId, 0); } @@ -256,6 +293,7 @@ public void reregistered(ExecutorDriver executorDriver, SlaveInfo slaveInfo) { @Override public void shutdown(ExecutorDriver executorDriver) { log.debug("shutting down executor"); + this.isShutDownInProgress = true; shutdownPlugin(); suicide(null, 0); } diff --git a/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java b/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java index 84ed468..5821999 100644 --- a/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java +++ b/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java @@ -23,7 +23,7 @@ public void init(DockerComposeExecutor executor, TaskID taskId) { @Override public void onCompleted() { - + log.info("DockerComposeProcessObserver: onCompleted "); } @Override @@ -34,7 +34,7 @@ public void onError(Throwable e) { @Override public void onNext(Integer t) { - log.info("executor for taskId:" + taskId.getValue() + " exited with exitCode:" + t); + log.info("DockerComposeProcessObserver: executor for taskId:" + taskId.getValue() + " exited with exitCode:" + t); executor.suicide(taskId, t); } From 23ffae5ae74afc299bdf31ab5d29682e5ce7ed94 Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Mon, 8 Aug 2016 10:16:20 -0700 Subject: [PATCH 07/15] Upgrading Mesos dependency to 0.28.2 Upgrading compiler plugin Setting version for maven assembly plug in in order to tone down noise --- pom.xml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 05f4c11..a2c9b8d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,10 +10,9 @@ org.apache.mesos mesos - 0.24.1 + 0.28.2 - log4j log4j @@ -79,16 +78,12 @@ test - - - - bin/classes maven-compiler-plugin - 3.1 + 3.5.1 com.google.dagger @@ -105,11 +100,11 @@ org.apache.maven.plugins maven-assembly-plugin + 2.6 jar-with-dependencies - From b531288e0bdda84efe90217e064cfe7f2f748a67 Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Mon, 8 Aug 2016 11:11:33 -0700 Subject: [PATCH 08/15] Fixed Manifest file so that fat jar can be run with java -jar instead of java -cp --- pom.xml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index a2c9b8d..6021fec 100644 --- a/pom.xml +++ b/pom.xml @@ -102,6 +102,12 @@ maven-assembly-plugin 2.6 + + + true + com.paypal.mesos.executor.App + + jar-with-dependencies @@ -119,14 +125,6 @@ org.apache.maven.plugins maven-jar-plugin - - - - true - com.paypal.mesos.executor.App - - - From fc0b18f36795378e2138d50f399237427e1db397 Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Mon, 8 Aug 2016 17:28:54 -0700 Subject: [PATCH 09/15] Applying a Maven workaround so that only the fat jar is in the dist folder after running mvn package --- pom.xml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index 6021fec..cf05583 100644 --- a/pom.xml +++ b/pom.xml @@ -4,58 +4,48 @@ com.paypal.mesos docker-compose-executor 0.0.1-SNAPSHOT - - org.apache.mesos mesos 0.28.2 - log4j log4j 1.2.17 - com.google.protobuf protobuf-java 2.5.0 - com.google.dagger dagger 2.0 - com.google.dagger dagger-compiler 2.0 true - org.apache.commons commons-lang3 3.0 - org.yaml snakeyaml 1.5 - com.google.code.gson gson 2.3.1 - io.reactivex rxjava @@ -110,7 +100,9 @@ jar-with-dependencies - + + docker-compose-executor + false @@ -126,7 +118,7 @@ org.apache.maven.plugins maven-jar-plugin - + + ${project.artifactId} - From eacadd8824d7b1f3c5247bbaec72a830ad406d5b Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Mon, 8 Aug 2016 17:32:05 -0700 Subject: [PATCH 10/15] Modified jar output file's name to be more standard, removed SNAPSHOT from name --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index cf05583..ea504e9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.paypal.mesos docker-compose-executor - 0.0.1-SNAPSHOT + 0.0.1 org.apache.mesos @@ -101,7 +101,7 @@ jar-with-dependencies - docker-compose-executor + docker-compose-executor_${project.version} false @@ -119,6 +119,6 @@ maven-jar-plugin - ${project.artifactId} + ${project.artifactId}_${project.version} From 6dae0cc2458ca9cef1eb3bd0efde0bd2c69ed5dc Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Mon, 8 Aug 2016 18:08:24 -0700 Subject: [PATCH 11/15] Modified the way in which Mesos and Marathon are installed. Updated documentation accordingly. \n\n Changed documentation to point to new sample tar.gz stored in bintray --- docs/getting-started.md | 16 ++- examples/vagrant/docker-compose-executor.sh | 5 +- examples/vagrant/provision-dev-cluster.sh | 32 +++-- examples/vagrant/provision-dev-cluster.sh_bak | 130 ------------------ 4 files changed, 30 insertions(+), 153 deletions(-) delete mode 100755 examples/vagrant/provision-dev-cluster.sh_bak diff --git a/docs/getting-started.md b/docs/getting-started.md index 52b4266..f6bca69 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -21,8 +21,7 @@ Marathon http://192.168.33.7:8080/ ``` -Create a marathon app -send a http post request to v2/apps +Create a marathon app by sending a JSON payload via HTTP to Marathon's app endpoint (192.168.33.7:8080/v2/apps) ``` { @@ -30,13 +29,18 @@ send a http post request to v2/apps "cmd": "echo hello world", "cpus": 1.0, "mem": 64.0, - "ports":[0,0,0], + "ports":[0], "instances": 1, - "executor":"/home/vagrant/aurora/examples/vagrant/docker-compose-executor.sh", + "executor":"/vagrant/examples/vagrant/docker-compose-executor.sh", "labels": { - "fileName": "web-app/docker-compose.yml" + "fileName": "sample-app/docker-compose.yml" }, - "uris":["https://dl.dropboxusercontent.com/u/26009359/web-app.zip"] + "uris":["https://dl.bintray.com/rdelvalle/mesos-compose-executor/sample-app.tar.gz"] } ``` +Using the curl command, a docker compose job can be created on Marathon as follows: +``` +$ curl -H "Content-Type: application/json" -X POST -d '{"id":"docker-compose-demo","cmd":"echo hello world","cpus":1.0,"mem":64.0,"ports":[0],"instances":1,"executor":"/vagrant/examples/vagrant/docker-compose-executor.sh","labels":{"fileName":"sample-app/docker-compose.yml"},"uris":["https://dl.bintray.com/rdelvalle/mesos-compose-executor/sample-app.tar.gz"]}' http://192.168.33.7:8080/v2/apps +``` + diff --git a/examples/vagrant/docker-compose-executor.sh b/examples/vagrant/docker-compose-executor.sh index b0eed60..140e03b 100755 --- a/examples/vagrant/docker-compose-executor.sh +++ b/examples/vagrant/docker-compose-executor.sh @@ -1,3 +1,2 @@ -COMPOSE_JAR_NAME=/home/vagrant/marathon/target/docker-compose-executor-0.0.1-SNAPSHOT-jar-with-dependencies.jar -COMPOSE_CLASS_NAME=com.paypal.mesos.executor.App -java -cp ${COMPOSE_JAR_NAME} ${COMPOSE_CLASS_NAME} +COMPOSE_JAR_NAME=/home/vagrant/marathon/target/docker-compose-executor_0.0.1.jar +java -jar ${COMPOSE_JAR_NAME} diff --git a/examples/vagrant/provision-dev-cluster.sh b/examples/vagrant/provision-dev-cluster.sh index 3836b4e..5c699b0 100755 --- a/examples/vagrant/provision-dev-cluster.sh +++ b/examples/vagrant/provision-dev-cluster.sh @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D +apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E56151BF +echo deb http://repos.mesosphere.com/ubuntu trusty main > /etc/apt/sources.list.d/mesosphere.list apt-get update -q --fix-missing apt-get -qy install software-properties-common @@ -37,7 +39,7 @@ apt-get -y install \ zookeeperd \ python-pip \ maven \ - build-essential \ + build-essential \ autoconf \ automake \ ca-certificates \ @@ -66,26 +68,26 @@ apt-get -y install \ update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java readonly IP_ADDRESS=192.168.33.7 - -readonly MESOS_VERSION=0.25.0 +readonly MESOS_VERSION=0.28.2-2.0.27 +readonly MARATHON_VERSION=1.1.2-1.0.482 function install_mesos { - deb=mesos_${MESOS_VERSION}-0.2.70.ubuntu1404_amd64.deb - wget -c http://downloads.mesosphere.io/master/ubuntu/14.04/$deb - dpkg --install $deb -} + apt-get -y install mesos=${MESOS_VERSION}.ubuntu1404 +} function install_marathon { - sudo pip install docker-compose - sudo dpkg --purge marathon - wget -c https://dl.dropboxusercontent.com/u/26009359/marathon_0.11.1-1.0.432.ubuntu1404_amd64.deb - sudo dpkg --install marathon_0.11.1-1.0.432.ubuntu1404_amd64.deb + apt-get -y install marathon=${MARATHON_VERSION}.ubuntu1404 +} + +function install_docker_compose { + pip install docker-compose } + function build_docker_compose_executor { - sudo mvn -f /home/vagrant/marathon/pom.xml clean package -U - sudo chmod 777 /home/vagrant/marathon/target/docker-compose-executor-0.0.1-SNAPSHOT-jar-with-dependencies.jar + mvn -f /home/vagrant/marathon/pom.xml clean package -U + chmod 777 /home/vagrant/marathon/target/docker-compose-executor_0.0.1.jar } function install_cluster_config { @@ -148,6 +150,8 @@ EOF } install_mesos +install_marathon +install_docker_compose prepare_sources install_marathon install_cluster_config diff --git a/examples/vagrant/provision-dev-cluster.sh_bak b/examples/vagrant/provision-dev-cluster.sh_bak deleted file mode 100755 index c453f26..0000000 --- a/examples/vagrant/provision-dev-cluster.sh_bak +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -ex -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 -echo deb https://get.docker.com/ubuntu docker main > /etc/apt/sources.list.d/docker.list - - -add-apt-repository ppa:openjdk-r/ppa -y -apt-get update -apt-get -y install \ - bison \ - curl \ - git \ - libapr1-dev \ - libcurl4-nss-dev \ - libsasl2-dev \ - libsvn-dev \ - lxc-docker \ - openjdk-8-jdk \ - python-dev \ - zookeeperd \ - python-pip \ - maven - -update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java - -readonly IP_ADDRESS=192.168.33.7 - -readonly MESOS_VERSION=0.25.0 - - -function install_mesos { - deb=mesos_${MESOS_VERSION}-0.2.70.ubuntu1404_amd64.deb - wget -c http://downloads.mesosphere.io/master/ubuntu/14.04/$deb - dpkg --install $deb -} - -function install_marathon { - sudo pip install docker-compose - sudo dpkg --purge marathon - wget -c https://dl.dropboxusercontent.com/u/26009359/marathon_0.11.1-1.0.432.ubuntu1404_amd64.deb - sudo dpkg --install marathon_0.11.1-1.0.432.ubuntu1404_amd64.deb -} - -function build_docker_compose_executor { - sudo mvn -f /home/vagrant/aurora/pom.xml clean package -U - sudo chmod 777 /home/vagrant/aurora/target/docker-compose-executor-0.0.1-SNAPSHOT-jar-with-dependencies.jar -} - -function install_cluster_config { - mkdir -p /etc/aurora - ln -sf /home/vagrant/aurora/examples/vagrant/clusters.json /etc/aurora/clusters.json -} - -function install_ssh_config { - cat >> /etc/ssh/ssh_config < /home/vagrant/.gradle/gradle.properties < /home/vagrant/.netrc < /usr/local/bin/update-sources < Date: Mon, 8 Aug 2016 18:11:42 -0700 Subject: [PATCH 12/15] Updating documentation to reflect changes made --- docs/dev-build.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/dev-build.md b/docs/dev-build.md index bc648a2..07d57d4 100644 --- a/docs/dev-build.md +++ b/docs/dev-build.md @@ -18,9 +18,8 @@ Use the fat jar generated with all the dependencies. > Marathon supports custom executor and you can specify a shell script (compose_executor.sh). ``` - COMPOSE_JAR_NAME=/docker-compose-executor-0.0.1-SNAPSHOT-jar-with-dependencies.jar - COMPOSE_CLASS_NAME=com.paypal.mesos.executor.App - java -cp ${COMPOSE_JAR_NAME} ${COMPOSE_CLASS_NAME} + COMPOSE_JAR_NAME=/docker-compose-executor_0.0.1.jar + java -jar ${COMPOSE_JAR_NAME} ``` > Create a new marathon app ``` From eb8dd2333d79296056645cb60eaa5e2afd58e773 Mon Sep 17 00:00:00 2001 From: Kumar Krishna Date: Tue, 9 Aug 2016 13:54:56 -0700 Subject: [PATCH 13/15] optimizing imports --- .../paypal/mesos/executor/CommandBuilder.java | 4 +-- .../mesos/executor/ComposeFileListModule.java | 7 ++--- .../mesos/executor/DockerComposeExecutor.java | 29 +++++++------------ .../DockerComposeProcessObserver.java | 1 - .../mesos/executor/ExecutorComponent.java | 5 ++-- .../paypal/mesos/executor/ExecutorModule.java | 10 +++---- .../executor/compose/ComposeFileList.java | 4 +-- .../executor/compose/ComposeFileListImpl.java | 13 ++++----- .../compose/ComposeRewriteHelper.java | 4 +-- .../executor/monitoring/ComposeMonitor.java | 20 ++++++------- .../pluginapi/ComposeExecutorPlugin.java | 1 + .../mesos/executor/utils/FileUtils.java | 14 ++++----- .../mesos/executor/utils/ProcessUtils.java | 10 ++----- .../compose/ComposeRewriteHelperTest.java | 9 ++---- 14 files changed, 52 insertions(+), 79 deletions(-) diff --git a/src/main/java/com/paypal/mesos/executor/CommandBuilder.java b/src/main/java/com/paypal/mesos/executor/CommandBuilder.java index 20c5755..cfea1ec 100644 --- a/src/main/java/com/paypal/mesos/executor/CommandBuilder.java +++ b/src/main/java/com/paypal/mesos/executor/CommandBuilder.java @@ -1,9 +1,9 @@ package com.paypal.mesos.executor; -import java.util.List; - import com.paypal.mesos.executor.config.Config; +import java.util.List; + public class CommandBuilder { diff --git a/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java b/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java index f321e4f..b6d874f 100644 --- a/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java +++ b/src/main/java/com/paypal/mesos/executor/ComposeFileListModule.java @@ -1,13 +1,12 @@ package com.paypal.mesos.executor; -import javax.inject.Singleton; - -import com.paypal.mesos.executor.compose.ComposeFileListImpl; import com.paypal.mesos.executor.compose.ComposeFileList; - +import com.paypal.mesos.executor.compose.ComposeFileListImpl; import dagger.Module; import dagger.Provides; +import javax.inject.Singleton; + @Module public class ComposeFileListModule { diff --git a/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java b/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java index fbbda43..9c86b8f 100644 --- a/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java +++ b/src/main/java/com/paypal/mesos/executor/DockerComposeExecutor.java @@ -1,25 +1,15 @@ package com.paypal.mesos.executor; -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import javax.inject.Inject; - +import com.paypal.mesos.executor.compose.ComposeFileList; import com.paypal.mesos.executor.compose.ComposeRewriteHelper; +import com.paypal.mesos.executor.monitoring.ComposeMonitor; +import com.paypal.mesos.executor.monitoring.ContainerDetails; import com.paypal.mesos.executor.pluginapi.ComposeExecutorPlugin; +import com.paypal.mesos.executor.utils.ProcessUtils; import org.apache.log4j.Logger; import org.apache.mesos.Executor; import org.apache.mesos.ExecutorDriver; -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.FrameworkInfo; -import org.apache.mesos.Protos.SlaveInfo; -import org.apache.mesos.Protos.TaskID; -import org.apache.mesos.Protos.TaskInfo; -import org.apache.mesos.Protos.TaskState; -import org.apache.mesos.Protos.TaskStatus; - +import org.apache.mesos.Protos.*; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; import ro.fortsoft.pf4j.DefaultPluginManager; @@ -29,10 +19,11 @@ import rx.functions.Action1; import rx.schedulers.Schedulers; -import com.paypal.mesos.executor.compose.ComposeFileList; -import com.paypal.mesos.executor.monitoring.ComposeMonitor; -import com.paypal.mesos.executor.monitoring.ContainerDetails; -import com.paypal.mesos.executor.utils.ProcessUtils; +import javax.inject.Inject; +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; public class DockerComposeExecutor implements Executor { diff --git a/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java b/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java index 5821999..171e301 100644 --- a/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java +++ b/src/main/java/com/paypal/mesos/executor/DockerComposeProcessObserver.java @@ -2,7 +2,6 @@ import org.apache.log4j.Logger; import org.apache.mesos.Protos.TaskID; - import rx.Observer; public class DockerComposeProcessObserver implements Observer { diff --git a/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java b/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java index 6d9b65e..fdbf071 100644 --- a/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java +++ b/src/main/java/com/paypal/mesos/executor/ExecutorComponent.java @@ -1,10 +1,9 @@ package com.paypal.mesos.executor; -import javax.inject.Singleton; - +import dagger.Component; import org.apache.mesos.Executor; -import dagger.Component; +import javax.inject.Singleton; @Singleton @Component(modules = {ComposeFileListModule.class, ExecutorModule.class}) diff --git a/src/main/java/com/paypal/mesos/executor/ExecutorModule.java b/src/main/java/com/paypal/mesos/executor/ExecutorModule.java index e4fb473..38a3605 100644 --- a/src/main/java/com/paypal/mesos/executor/ExecutorModule.java +++ b/src/main/java/com/paypal/mesos/executor/ExecutorModule.java @@ -1,18 +1,16 @@ package com.paypal.mesos.executor; -import javax.inject.Singleton; - -import com.paypal.mesos.executor.compose.ComposeRewriteHelper; -import org.apache.mesos.Executor; - import com.paypal.mesos.executor.compose.ComposeFileList; +import com.paypal.mesos.executor.compose.ComposeRewriteHelper; import com.paypal.mesos.executor.monitoring.ComposeMonitor; - import dagger.Module; import dagger.Provides; +import org.apache.mesos.Executor; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; +import javax.inject.Singleton; + @Module public class ExecutorModule { diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java index d125b47..a8cb68e 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileList.java @@ -1,10 +1,10 @@ package com.paypal.mesos.executor.compose; +import org.apache.mesos.Protos.TaskInfo; + import java.io.IOException; import java.util.List; -import org.apache.mesos.Protos.TaskInfo; - public interface ComposeFileList { List getFile(TaskInfo taskInfo) throws IOException; diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java index 1c65047..5e6df0f 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java @@ -1,18 +1,17 @@ package com.paypal.mesos.executor.compose; +import org.apache.log4j.Logger; +import org.apache.mesos.Protos.Label; +import org.apache.mesos.Protos.Labels; +import org.apache.mesos.Protos.TaskInfo; + +import javax.inject.Inject; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; import java.util.List; -import javax.inject.Inject; - -import org.apache.log4j.Logger; -import org.apache.mesos.Protos.Label; -import org.apache.mesos.Protos.Labels; -import org.apache.mesos.Protos.TaskInfo; - public class ComposeFileListImpl implements ComposeFileList { diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java index 6e54952..5690c1b 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeRewriteHelper.java @@ -1,7 +1,5 @@ package com.paypal.mesos.executor.compose; -import java.util.*; - import org.apache.log4j.Logger; import org.apache.mesos.Protos; import org.apache.mesos.Protos.ExecutorInfo; @@ -10,6 +8,8 @@ import org.apache.mesos.Protos.Value.Range; import org.apache.mesos.Protos.Value.Ranges; +import java.util.*; + public class ComposeRewriteHelper { diff --git a/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java b/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java index a931eeb..89717a4 100644 --- a/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java +++ b/src/main/java/com/paypal/mesos/executor/monitoring/ComposeMonitor.java @@ -1,23 +1,21 @@ package com.paypal.mesos.executor.monitoring; -import java.io.ByteArrayOutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.StringTokenizer; -import java.util.concurrent.TimeUnit; - +import com.paypal.mesos.executor.CommandBuilder; +import com.paypal.mesos.executor.config.Config; +import com.paypal.mesos.executor.utils.ProcessUtils; import org.apache.log4j.Logger; - import rx.Observable; import rx.Observer; import rx.Subscription; import rx.functions.Action1; import rx.subjects.BehaviorSubject; -import com.paypal.mesos.executor.CommandBuilder; -import com.paypal.mesos.executor.config.Config; -import com.paypal.mesos.executor.utils.ProcessUtils; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.StringTokenizer; +import java.util.concurrent.TimeUnit; //TODO remode public class ComposeMonitor { diff --git a/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java index e41c532..d7a9b66 100644 --- a/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java +++ b/src/main/java/com/paypal/mesos/executor/pluginapi/ComposeExecutorPlugin.java @@ -3,6 +3,7 @@ import org.apache.mesos.ExecutorDriver; import org.apache.mesos.Protos; import ro.fortsoft.pf4j.ExtensionPoint; + import java.util.List; /** diff --git a/src/main/java/com/paypal/mesos/executor/utils/FileUtils.java b/src/main/java/com/paypal/mesos/executor/utils/FileUtils.java index c85de61..113bf9e 100644 --- a/src/main/java/com/paypal/mesos/executor/utils/FileUtils.java +++ b/src/main/java/com/paypal/mesos/executor/utils/FileUtils.java @@ -1,19 +1,15 @@ package com.paypal.mesos.executor.utils; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; +import org.apache.log4j.Logger; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; + +import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import org.apache.log4j.Logger; -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; - public class FileUtils { private static final Logger log = Logger.getLogger(FileUtils.class); diff --git a/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java b/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java index 26a24cf..32402b8 100644 --- a/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java +++ b/src/main/java/com/paypal/mesos/executor/utils/ProcessUtils.java @@ -1,17 +1,13 @@ package com.paypal.mesos.executor.utils; +import org.apache.commons.exec.*; +import org.apache.log4j.Logger; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.concurrent.TimeUnit; -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.exec.OS; -import org.apache.commons.exec.PumpStreamHandler; -import org.apache.log4j.Logger; - public class ProcessUtils { private static Logger log = Logger.getLogger(ProcessUtils.class); diff --git a/src/test/java/com/paypal/mesos/executor/compose/ComposeRewriteHelperTest.java b/src/test/java/com/paypal/mesos/executor/compose/ComposeRewriteHelperTest.java index 4dce11d..79e516c 100644 --- a/src/test/java/com/paypal/mesos/executor/compose/ComposeRewriteHelperTest.java +++ b/src/test/java/com/paypal/mesos/executor/compose/ComposeRewriteHelperTest.java @@ -3,15 +3,12 @@ import com.paypal.mesos.executor.DockerComposeExecutor; import com.paypal.mesos.executor.DockerComposeProcessObserver; import com.paypal.mesos.executor.monitoring.ComposeMonitor; -import junit.framework.Test; import org.apache.mesos.Protos; -import com.paypal.mesos.executor.compose.ComposeRewriteHelper; import org.junit.Assert; -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; -import java.io.*; -import java.util.*; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; /** From 649a9cd6fb4edd2bf99ca1ef7a1dd4d4348c86ff Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Mon, 8 Aug 2016 18:54:40 -0700 Subject: [PATCH 14/15] Adding a version to maven-jar-plugin --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea504e9..d46ef93 100644 --- a/pom.xml +++ b/pom.xml @@ -116,7 +116,8 @@ org.apache.maven.plugins - maven-jar-plugin + maven-jar-plugin + 3.0.2 ${project.artifactId}_${project.version} From 356ccfcd82e34723d7800cb1ceb29ec5c17e977b Mon Sep 17 00:00:00 2001 From: Kumar Krishna Date: Tue, 9 Aug 2016 14:17:01 -0700 Subject: [PATCH 15/15] for compatibility with aurora client --- .../com/paypal/mesos/executor/compose/ComposeFileListImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java index 5e6df0f..dc9db21 100644 --- a/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java +++ b/src/main/java/com/paypal/mesos/executor/compose/ComposeFileListImpl.java @@ -40,7 +40,8 @@ private List getFileName(TaskInfo taskInfo) { Labels labels = taskInfo.getLabels(); for (Label label : labels.getLabelsList()) { - if ("fileName".equals(label.getKey())) { + //Aurora client driver uses keyname - "org.apache.aurora.metadata.fileName".. + if (label.getKey().endsWith("fileName")) { List files = Arrays.asList(label.getValue().split(FILE_DELIMITER)); for (int i = 0; i < files.size(); i++) files.set(i, files.get(i).trim());