From 9a7b8e1e00191a6dadd23c8d9ad011409c4ebee4 Mon Sep 17 00:00:00 2001 From: alfoa Date: Wed, 20 Jan 2021 16:14:53 -0700 Subject: [PATCH 01/57] derivatives --- framework/Models/ROM.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 3d67deae6d..825ed2922c 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -42,7 +42,6 @@ class ROM(Dummy): """ ROM stands for Reduced Order Model. All the models here, first learn than predict the outcome """ - @classmethod def getInputSpecification(cls): """ @@ -1432,6 +1431,25 @@ def evaluate(self, request): outputEvaluation[k] = np.atleast_1d(v) return outputEvaluation + def derivatives(self, request, feats = None, order="first"): + """ + This method is aimed to evaluate the derivatives using this ROM + The differentiation method depends on the specific ROM. Numerical + differentiation is available by default for any ROM. Some ROMs (e.g. + neural networks) use automatic differentiation. + @ In, request, dict, coordinate of the central point + @ In, feats, list, optional, list of features we need to compute the + derivative for (default all) + @ In, order, str, order of the derivative (first, second) + @ Out, derivatives, dict, the dict containing the derivatives for each target + ({'d(feature1)/d(target1)':np.array(size 1 or n_ts), + 'd(feature1)/d(target1)':np.array(...)}. For example, + {"x1/y1":np.array(...),"x2/y1":np.array(...),etc.} + """ + inputToROM = self._inputToInternal(request) + derivatives = self.supervisedEngine.derivatives(inputToROM,order,feats) + return derivatives + def _externalRun(self,inRun): """ Method that performs the actual run of the imported external model (separated from run method for parallelization purposes) From 4600e131f1071d35945efbc8c014e172ae416168 Mon Sep 17 00:00:00 2001 From: alfoa Date: Thu, 21 Jan 2021 12:47:10 -0700 Subject: [PATCH 02/57] added derivative in mathUtils --- framework/utils/mathUtils.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/framework/utils/mathUtils.py b/framework/utils/mathUtils.py index 952a9a003d..950dfe8596 100644 --- a/framework/utils/mathUtils.py +++ b/framework/utils/mathUtils.py @@ -962,6 +962,37 @@ def angleBetweenVectors(a, b): ang = np.rad2deg(ang) return ang +def derivative(f, x0, var, n = 1, h = 1e-6, order = None): + """ + Compute the n-th partial derivative of function f + with respect variable var (numerical differentation). + The derivative is computed with a central difference + approximation. + @ In, f, instance, the function to differentiate (format f(d) where d is a dictionary) + @ In, x0, dict, the dictionary containing the x0 coordinate + @ In, var, str, the variable of the resulting partial derivative + @ In, n, int, optional, the order of the derivative. If n>2, the order param must be inputted + @ In, h, float, the step size + @ In, order, int, the order (n points) to compute the derivative (required if n>2) + @ Out, deriv, float, the partial derivative of function f + """ + from scipy.misc import derivative as dev + def func(x, var): + """ + Simple function wrapper for using scipy + @ In, x, float, the point at which the nth derivative is found + @ In, var, str, the variable in the dictionary x0 corresponding + to the part derivative to compute + @ Out, func, float, the evaluated function + """ + d = copy.copy(x0) + d[var] = x + return f(d) + if not order: + assert(n <= 2) + deriv = dev(func, x0[var], dx=h, n=n, args=(var, ), order=order if order else 3) + return deriv + # utility function for defaultdict def giveZero(): """ From 1f96c910aadd3af94657d601dfd829b5d3890701 Mon Sep 17 00:00:00 2001 From: alfoa Date: Tue, 16 Mar 2021 12:56:48 -0600 Subject: [PATCH 03/57] deriv --- dependencies.xml | 1 + framework/utils/mathUtils.py | 31 +++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/dependencies.xml b/dependencies.xml index 7f9150867b..962a27a556 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -57,6 +57,7 @@ Note all install methods after "main" take 6.0 1.1 + 0.9.39 remove diff --git a/framework/utils/mathUtils.py b/framework/utils/mathUtils.py index eac7c065ce..f8aabd5aba 100644 --- a/framework/utils/mathUtils.py +++ b/framework/utils/mathUtils.py @@ -962,7 +962,7 @@ def angleBetweenVectors(a, b): ang = np.rad2deg(ang) return ang -def derivative(f, x0, var, n = 1, h = 1e-6, order = None): +def derivative(f, x0, var, n = 1, h = None): """ Compute the n-th partial derivative of function f with respect variable var (numerical differentation). @@ -971,12 +971,12 @@ def derivative(f, x0, var, n = 1, h = 1e-6, order = None): @ In, f, instance, the function to differentiate (format f(d) where d is a dictionary) @ In, x0, dict, the dictionary containing the x0 coordinate @ In, var, str, the variable of the resulting partial derivative - @ In, n, int, optional, the order of the derivative. If n>2, the order param must be inputted - @ In, h, float, the step size - @ In, order, int, the order (n points) to compute the derivative (required if n>2) + @ In, n, int, optional, the order of the derivative. (max 10) + @ In, h, float, optional, the step size, default automatically computed @ Out, deriv, float, the partial derivative of function f """ - from scipy.misc import derivative as dev + import numdifftools as nd + assert(n <= 10) def func(x, var): """ Simple function wrapper for using scipy @@ -988,9 +988,24 @@ def func(x, var): d = copy.copy(x0) d[var] = x return f(d) - if not order: - assert(n <= 2) - deriv = dev(func, x0[var], dx=h, n=n, args=(var, ), order=order if order else 3) + do = nd.Derivative(func, order=n) + deriv = do(x0[var],var) + + #from scipy.misc import derivative as dev + #def func(x, var): + # """ + # Simple function wrapper for using scipy + # @ In, x, float, the point at which the nth derivative is found + # @ In, var, str, the variable in the dictionary x0 corresponding + # to the part derivative to compute + # @ Out, func, float, the evaluated function + # """ + # d = copy.copy(x0) + # d[var] = x + # return f(d) + #if not order: + # assert(n <= 2) + #deriv = dev(func, x0[var], dx=h, n=n, args=(var, ), order=order if order else 3) return deriv # utility function for defaultdict From 7ce269b97a17a453e8110812c899744b192649b4 Mon Sep 17 00:00:00 2001 From: alfoa Date: Wed, 17 Mar 2021 10:30:05 -0600 Subject: [PATCH 04/57] added export of pyomo --- framework/LearningGate.py | 30 +++++++++ framework/Models/ROM.py | 121 ++++++++++++++++++++++++++++++++++- framework/utils/mathUtils.py | 65 +++++++++++++------ 3 files changed, 192 insertions(+), 24 deletions(-) diff --git a/framework/LearningGate.py b/framework/LearningGate.py index 4085c4c08e..f3de467f06 100644 --- a/framework/LearningGate.py +++ b/framework/LearningGate.py @@ -289,6 +289,36 @@ def evaluate(self,request): resultsDict[key] = np.append(resultsDict[key],sliceEvaluation[key]) return resultsDict + def derivatives(self, request, order=1, feats=None): + """ + Method to perform the evaluation of the derivatives around a point or a set of points through the linked surrogate model + @ In, request, dict, realizations request ({'feature1':np.array(n_realizations),'feature2',np.array(n_realizations)}) + @ In, order, int, optional, the order of the derivatives (1 =first, 2=second, etc.). default 1. + @ In, feats, list, optional, list of features we need to compute the + derivative for (default all) + @ Out, resultsDict, dict, the dict containing the derivatives for each target + ({'d(feature1)|d(target1)':np.array(size 1 or n_ts), + 'd(feature1)|d(target1)':np.array(...)}. For example, + {"dx1|dy1":np.array(...),"dx2|dy1":np.array(...),etc.} + """ + if self.pickled: + self.raiseAnError(RuntimeError,'ROM "'+self.initializationOptions['name']+'" has not been loaded yet! Use an IOStep to load it.') + if not self.amITrained: + self.raiseAnError(RuntimeError, "ROM "+self.initializationOptions['name']+" has not been trained yet and, consequentially, can not be evaluated for derivative enstimation!") + resultsDict = {} + if isinstance(self.supervisedContainer[0], SupervisedLearning.Collection): + resultsDict = mathUtils.derivatives(self.supervisedContainer[0].evaluate, request, var=feats, n=order) + else: + for rom in self.supervisedContainer: + sliceEvaluation = mathUtils.derivatives(rom.evaluate, request, var=feats, n=order) + if len(list(resultsDict.keys())) == 0: + resultsDict.update(sliceEvaluation) + else: + for key in resultsDict.keys(): + resultsDict[key] = np.append(resultsDict[key],sliceEvaluation[key]) + return resultsDict + + __interfaceDict = {} __interfaceDict['SupervisedGate' ] = supervisedLearningGate __base = 'supervisedGate' diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 484856a7ba..10d345a925 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -1469,7 +1469,7 @@ def evaluate(self, request): outputEvaluation[k] = np.atleast_1d(v) return outputEvaluation - def derivatives(self, request, feats = None, order="first"): + def derivatives(self, request, feats = None, order=1): """ This method is aimed to evaluate the derivatives using this ROM The differentiation method depends on the specific ROM. Numerical @@ -1478,14 +1478,16 @@ def derivatives(self, request, feats = None, order="first"): @ In, request, dict, coordinate of the central point @ In, feats, list, optional, list of features we need to compute the derivative for (default all) - @ In, order, str, order of the derivative (first, second) + @ In, order, int, order of the derivative (1 = first, 2 = second, etc.). Max 10 @ Out, derivatives, dict, the dict containing the derivatives for each target ({'d(feature1)/d(target1)':np.array(size 1 or n_ts), 'd(feature1)/d(target1)':np.array(...)}. For example, {"x1/y1":np.array(...),"x2/y1":np.array(...),etc.} """ inputToROM = self._inputToInternal(request) - derivatives = self.supervisedEngine.derivatives(inputToROM,order,feats) + derivatives = self.supervisedEngine.derivatives(inputToROM, order=order, feats=feats) + for k,v in derivatives.items(): + derivatives[k] = np.atleast_1d(v) return derivatives def _externalRun(self,inRun): @@ -1640,3 +1642,116 @@ def writeXML(self, what='all'): engines[0].writeXMLPreamble(xml) engines[0].writeXML(xml) return xml + + def writePyomoGreyModel(self): + """ + Called by the OutStreamPrint object to cause the ROM to print itself + @ In, what, string, optional, keyword requesting what should be printed + @ Out, xml, xmlUtils.StaticXmlElement, written meta + """ + template = r""" + # MODEL GENERATED BY RAVEN (raven.inl.gov) + import pyomo.environ as pyo + from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock + from pyomo.contrib.pynumero.dependencies import (numpy as np, scipy_sparse as spa) + from pyomo.contrib.pynumero.asl import AmplInterface + from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP, ipopt, ipopt_available + import os, sys + # RAVEN ROM PYOMO GREY MODEL CLASS + + class ravenROM(ExternalGreyBoxModel): + + def __init__(self, **kwargs): + self._rom_file = kwargs.get("rom_file") + self._raven_framework = kwargs.get("raven_framework") + self._raven_framework = os.path.abspath(self._raven_framework) + if not os.path.exists(self._raven_framework): + raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') + if os.path.dirname(self._raven_framework).endswith("framework"): + sys.path.append(self._raven_framework) + else: + sys.path.append(os.path.join(self._raven_framework,"framework")) + import Driver + # de-serialize the ROM + self._rom_file = os.path.abspath(self._rom_file) + if not os.path.exists(self._rom_file): + raise IOError('The serialized (binary) file has not been found in location "' + str(self._rom_file)+'" !') + self.rom = pickle.load(open(self._rom_file, mode='rb')) + # + self.settings = self.rom.getInitParams() + # get input names + self._input_names = self.settings.get('Features') + # n_inputs + self._n_inputs = len(self._input_names) + # get output names + self._output_names = self.settings.get('Target') + # n_outputs + self._n_outputs = len(self._output_names) + # input values storage + self._input_values = np.zeros(self._n_inputs, dtype=np.float64) + + def return_train_values(self, feat): + return self.rom.trainingSet.get(feat) + + + def input_names(self): + return self._input_names + + def output_names(self): + return self._output_names + + def set_input_values(self, input_values): + assert len(input_values) == self._n_inputs + np.copyto(self._input_values, input_values) + + def evaluate_equality_constraints(self): + raise NotImplementedError('This method should not be called for this model.') + + def evaluate_outputs(self): + request = {k,np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + outs = self.rom.evaluate(request) + return np.asaarray([outs[k].flatten() for k in self._output_names], dtype=np.float6) + + def evaluate_jacobian_outputs(self): + request = {k,np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + derivatives = self.rom.derivatives(request) + jac = np.zeros((self._n_outputs, self._n_inputs)) + for tc, target in enumerate(self._output_names): + for fc, feature in enumerate(self._input_names): + jac[tc,fc] = derivatives['d{}|d{}'.format(feature,target)] + return jac + + def pyomoModel(ex_model): + m = pyo.ConcreteModel() + m.egb = ExternalGreyBoxBlock() + m.egb.set_external_model(ex_model) + for inp in ex_model.input_names(): + m.egb.inputs[inp].value = np.mean(ex_model.return_train_values(inp)) + m.egb.inputs[inp].setlb(np.min(ex_model.return_train_values(inp))) + m.egb.inputs[inp].setub(np.max(ex_model.return_train_values(inp))) + for out in ex_model.output_names(): + m.egb.outputs[out].value = np.mean(ex_model.return_train_values(out)) + m.egb.outputs[out].setlb(np.min(ex_model.return_train_values(out))) + m.egb.outputs[out].setub(np.max(ex_model.return_train_values(out))) + m.obj = pyo.Objective(expr=m.egb.outputs[out]) + + return m + + if __name__ == '__main__': + for cnt, item in enumerate(sys.argv): + if item.lower("-r"): + rom_file = sys.argv[cnt+1] + if item.lower("-f"): + raven_framework = sys.argv[cnt+1] + ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework}) + concreteModel = pyomoModel(ext_model) + ### here you should implement the optimization problem + ### + + + """ + + return template + + + diff --git a/framework/utils/mathUtils.py b/framework/utils/mathUtils.py index f8aabd5aba..1179525beb 100644 --- a/framework/utils/mathUtils.py +++ b/framework/utils/mathUtils.py @@ -962,9 +962,9 @@ def angleBetweenVectors(a, b): ang = np.rad2deg(ang) return ang -def derivative(f, x0, var, n = 1, h = None): +def partialDerivative(f, x0, var, n = 1, h = None, target = None): """ - Compute the n-th partial derivative of function f + Compute the n-th partial derivative of function f with respect variable var (numerical differentation). The derivative is computed with a central difference approximation. @@ -973,41 +973,64 @@ def derivative(f, x0, var, n = 1, h = None): @ In, var, str, the variable of the resulting partial derivative @ In, n, int, optional, the order of the derivative. (max 10) @ In, h, float, optional, the step size, default automatically computed + @ In, target, str, optional, the target output in case the "f" returns a dict of outputs. Default (takes the first) @ Out, deriv, float, the partial derivative of function f """ import numdifftools as nd assert(n <= 10) - def func(x, var): + def func(x, var, target=None): """ Simple function wrapper for using scipy @ In, x, float, the point at which the nth derivative is found @ In, var, str, the variable in the dictionary x0 corresponding to the part derivative to compute + @ In, target, str, optional, the target output in case the "f" returns a dict of outputs. Default (takes the first) @ Out, func, float, the evaluated function """ d = copy.copy(x0) d[var] = x - return f(d) + out = f(d) + if isinstance(out, dict): + return list(out.values())[0] if target is None else out[target] + else: + return out + do = nd.Derivative(func, order=n) - deriv = do(x0[var],var) - - #from scipy.misc import derivative as dev - #def func(x, var): - # """ - # Simple function wrapper for using scipy - # @ In, x, float, the point at which the nth derivative is found - # @ In, var, str, the variable in the dictionary x0 corresponding - # to the part derivative to compute - # @ Out, func, float, the evaluated function - # """ - # d = copy.copy(x0) - # d[var] = x - # return f(d) - #if not order: - # assert(n <= 2) - #deriv = dev(func, x0[var], dx=h, n=n, args=(var, ), order=order if order else 3) + deriv = do(x0[var],var,target) + return deriv + +def derivatives(f, x0, var = None, n = 1, h = None, target=None): + """ + Compute the n-th partial derivative of function f + with respect variable var (numerical differentation). + The derivative is computed with a central difference + approximation. + @ In, f, instance, the function to differentiate (format f(d) where d is a dictionary) + @ In, x0, dict, the dictionary containing the x0 coordinate {key:scalar or array of len(1)} + @ In, var, list, optional, the list of variables of the resulting partial derivative (if None, compute all) + @ In, n, int, optional, the order of the derivative. (max 10) + @ In, h, float, optional, the step size, default automatically computed + @ In, target, str, optional, the target output in case the "f" returns a dict of outputs. Default (all targets) + @ Out, deriv, dict, the partial derivative of function f + """ + assert(n <= 10) + assert(isinstance(var, list) or isinstance(var, type(None))) + assert(len(list(x0.values())[0]) == 1) + targets = [target] + if target is None: + checkWorking = f(x0) + if isinstance(checkWorking, dict): + targets = checkWorking.keys() + deriv = {} + for t in targets: + for variable in (x0.keys() if var is None else var): + name = variable if t is None else "d{}|d{}".format(t, variable) + deriv[name] = partialDerivative(f, x0, variable, n = n, h = h, target=t) return deriv + + + # utility function for defaultdict def giveZero(): """ From 064ed7cbc907a9ebd291c7cf9ed4d3112f88f48e Mon Sep 17 00:00:00 2001 From: alfoa Date: Wed, 17 Mar 2021 11:41:34 -0600 Subject: [PATCH 05/57] moving --- framework/Models/ROM.py | 1 - 1 file changed, 1 deletion(-) diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 10d345a925..decd97d0dd 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -1693,7 +1693,6 @@ def __init__(self, **kwargs): def return_train_values(self, feat): return self.rom.trainingSet.get(feat) - def input_names(self): return self._input_names From 109029abd8b106630b8a06985d8e5b01c46d40e3 Mon Sep 17 00:00:00 2001 From: alfoa Date: Mon, 5 Apr 2021 10:52:20 -0600 Subject: [PATCH 06/57] model --- framework/Models/Model.py | 26 +++++++++++++++++++++----- framework/Steps.py | 11 +++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/framework/Models/Model.py b/framework/Models/Model.py index 73bed31a80..059aa0d1e0 100644 --- a/framework/Models/Model.py +++ b/framework/Models/Model.py @@ -13,11 +13,10 @@ # limitations under the License. """ Module where the base class and the specialization of different type of Model are -""" -#for future compatibility with Python 3-------------------------------------------------------------- -from __future__ import division, print_function, unicode_literals, absolute_import -#End compatibility block for Python 3---------------------------------------------------------------- +@author crisrab, alfoa + +""" #External Modules------------------------------------------------------------------------------------ import copy import numpy as np @@ -48,7 +47,6 @@ def loadFromPlugins(cls): """ cls.plugins = importlib.import_module("Models.ModelPlugInFactory") - @classmethod def getInputSpecification(cls): """ @@ -352,9 +350,27 @@ def initialize(self,runInfo,inputs,initDict=None): @ In, runInfo, dict, it is the run info from the jobHandler @ In, inputs, list, it is a list containing whatever is passed with an input role in the step @ In, initDict, dict, optional, dictionary of all objects available in the step is using this model + @ In, None """ pass + def serialize(self,fileo,**kwargs): + """ + This method is the base class method that is aimed to serialize the model (and derived) instances. + @ In, fileo, str or File object, the filename of the output serialized binary file or the RAVEN File instance + @ In, kwargs, dict, dictionary of options that the derived class might require + @ Out, None + """ + import cloudpickle + if isinstance(fileo,str): + fileobj = open(filename, mode='wb+') + else: + fileobj = fileo + fileobj.open(mode='wb+') + cloudpickle.dump(self,fileobj) + fileobj.flush() + fileobj.close() + @abc.abstractmethod def createNewInput(self,myInput,samplerType,**kwargs): """ diff --git a/framework/Steps.py b/framework/Steps.py index 3a881cc164..0229ef164b 100644 --- a/framework/Steps.py +++ b/framework/Steps.py @@ -1079,14 +1079,13 @@ def _localTakeAstepRun(self,inDictionary): elif self.actionType[i] == 'ROM-FILES': #inDictionary['Input'][i] is a ROM, outputs[i] is Files ## pickle the ROM - #check the ROM is trained first + # check the ROM is trained first if not inDictionary['Input'][i].amITrained: self.raiseAnError(RuntimeError,'Pickled rom "%s" was not trained! Train it before pickling and unpickling using a RomTrainer step.' %inDictionary['Input'][i].name) - fileobj = outputs[i] - fileobj.open(mode='wb+') - cloudpickle.dump(inDictionary['Input'][i],fileobj) - fileobj.flush() - fileobj.close() + # call the serialize method within the model + ## TODO: ADD Deserialization method + inDictionary['Input'][i].serialize(outputs[i]) + elif self.actionType[i] == 'FILES-ROM': #inDictionary['Input'][i] is a Files, outputs[i] is ROM ## unpickle the ROM From feb0bcafefd172075063504134a0c1bf178807c3 Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Wed, 11 Aug 2021 16:20:16 -0600 Subject: [PATCH 07/57] Fixing bug. --- framework/Models/Model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/framework/Models/Model.py b/framework/Models/Model.py index 373511c781..d9e8f07e4c 100644 --- a/framework/Models/Model.py +++ b/framework/Models/Model.py @@ -23,6 +23,7 @@ import abc import sys import importlib +import pickle #External Modules End-------------------------------------------------------------------------------- #Internal Modules------------------------------------------------------------------------------------ @@ -397,7 +398,7 @@ def serialize(self,fileo,**kwargs): else: fileobj = fileo fileobj.open(mode='wb+') - cloudpickle.dump(self,fileobj) + cloudpickle.dump(self,fileobj, protocol=pickle.HIGHEST_PROTOCOL) fileobj.flush() fileobj.close() From f80c989f07d52305c977939d9f1a48f8e84b545f Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Wed, 18 Aug 2021 15:58:21 -0600 Subject: [PATCH 08/57] Adding a test that pyomo serialization can write a file. --- framework/Steps/IOStep.py | 12 ++- tests/framework/.gitignore | 4 +- .../Models/External/serialize_pyomo.xml | 78 +++++++++++++++++++ tests/framework/Models/External/tests | 5 ++ 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 tests/framework/Models/External/serialize_pyomo.xml diff --git a/framework/Steps/IOStep.py b/framework/Steps/IOStep.py index 08c85e53c8..842fd42ef5 100644 --- a/framework/Steps/IOStep.py +++ b/framework/Steps/IOStep.py @@ -121,7 +121,12 @@ def _localInitializeStep(self,inDictionary): elif isinstance(inDictionary['Input'][i], (Models.ROM, Models.ExternalModel)): # ... file if isinstance(outputs[i],Files.File): - self.actionType.append('MODEL-FILES') + #XXX switching on the file extension is not very + # extendable. Possibly should have a type or something + if 'py' == outputs[i].getExt().lower(): + self.actionType.append('MODEL-PYOMO') + else: + self.actionType.append('MODEL-FILES') # ... data object elif isinstance(outputs[i], DataObject.DataObject): self.actionType.append('ROM-dataObjects') @@ -218,6 +223,11 @@ def _localTakeAstepRun(self,inDictionary): ## TODO: ADD Deserialization method inDictionary['Input'][i].serialize(outputs[i]) + elif self.actionType[i] == 'MODEL-PYOMO': + outfile = open(outputs[i].getAbsFile(),"w") + outfile.write(inDictionary['Input'][i].writePyomoGreyModel()) + outfile.close() + elif self.actionType[i] == 'FILES-MODEL': #inDictionary['Input'][i] is a Files, outputs[i] is ROM or ExternalModel ## unpickle the ROM diff --git a/tests/framework/.gitignore b/tests/framework/.gitignore index 8539e3ce56..a3b05df6ee 100644 --- a/tests/framework/.gitignore +++ b/tests/framework/.gitignore @@ -179,4 +179,6 @@ CodeInterfaceTests/MOOSEBaseApps/MooseExample18WithGenericFile/myMC/ex18.i CodeInterfaceTests/MOOSEBaseApps/InputParser/sample/[12]/formattest.i CodeInterfaceTests/MOOSEBaseApps/InputParser/sample/[12]/out~formattest CodeInterfaceTests/MOOSEBaseApps/InputParser/sample/formattest.i -hybridModel/logicalCode/logicalModelCode/ \ No newline at end of file +hybridModel/logicalCode/logicalModelCode/ +Models/External/SerializePyomo/ +Models/External/SerializeWrkd/ diff --git a/tests/framework/Models/External/serialize_pyomo.xml b/tests/framework/Models/External/serialize_pyomo.xml new file mode 100644 index 0000000000..fd3e33605b --- /dev/null +++ b/tests/framework/Models/External/serialize_pyomo.xml @@ -0,0 +1,78 @@ + + + + + + + SerializePyomo + sample, train, serialize + + + + rom_out.py + + + + + dummyIN + attenuate + grid + samples + + + samples + out_rom + + + out_rom + rom_out.py + + + + + + y1,y2 + ans,fromInit,fromReadMoreXML,fromCNISelf,fromCNIDict + + 3.14159 + + + + linear_model|LinearRegression + y1,y2 + ans + + + + + + 0 + 1 + + + + + + + dist + 0 1 + + + dist + 0 1 + + + + + + + y1,y2 + + + y1, y2 + ans + + + + + diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index 4a2a31e030..d95eae2284 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -9,6 +9,11 @@ input = 'serialize_ext_model_and_use.xml' csv = 'SerializeWrkd/samples_out.csv' [../] + [./pyomo_serialization] + type = 'RavenFramework' + input = 'serialize_pyomo.xml' + output = 'SerializePyomo/rom_out.py' + [../] [] From 5bd27c9945ef664667a11eafd4325af649e72483 Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Mon, 23 Aug 2021 10:35:34 -0600 Subject: [PATCH 09/57] Fixing indent in template string. --- framework/Models/ROM.py | 172 ++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 87 deletions(-) diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 41119c9692..0af4202d2a 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -1677,105 +1677,103 @@ def writePyomoGreyModel(self): @ Out, xml, xmlUtils.StaticXmlElement, written meta """ template = r""" - # MODEL GENERATED BY RAVEN (raven.inl.gov) - import pyomo.environ as pyo - from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock - from pyomo.contrib.pynumero.dependencies import (numpy as np, scipy_sparse as spa) - from pyomo.contrib.pynumero.asl import AmplInterface - from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP, ipopt, ipopt_available - import os, sys - # RAVEN ROM PYOMO GREY MODEL CLASS +# MODEL GENERATED BY RAVEN (raven.inl.gov) +import pyomo.environ as pyo +from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock +from pyomo.contrib.pynumero.dependencies import (numpy as np, scipy_sparse as spa) +from pyomo.contrib.pynumero.asl import AmplInterface +from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP, ipopt, ipopt_available +import os, sys +# RAVEN ROM PYOMO GREY MODEL CLASS - class ravenROM(ExternalGreyBoxModel): +class ravenROM(ExternalGreyBoxModel): - def __init__(self, **kwargs): - self._rom_file = kwargs.get("rom_file") - self._raven_framework = kwargs.get("raven_framework") - self._raven_framework = os.path.abspath(self._raven_framework) - if not os.path.exists(self._raven_framework): - raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') - if os.path.dirname(self._raven_framework).endswith("framework"): - sys.path.append(self._raven_framework) - else: - sys.path.append(os.path.join(self._raven_framework,"framework")) - import Driver - # de-serialize the ROM - self._rom_file = os.path.abspath(self._rom_file) - if not os.path.exists(self._rom_file): - raise IOError('The serialized (binary) file has not been found in location "' + str(self._rom_file)+'" !') - self.rom = pickle.load(open(self._rom_file, mode='rb')) - # - self.settings = self.rom.getInitParams() - # get input names - self._input_names = self.settings.get('Features') - # n_inputs - self._n_inputs = len(self._input_names) - # get output names - self._output_names = self.settings.get('Target') - # n_outputs - self._n_outputs = len(self._output_names) - # input values storage - self._input_values = np.zeros(self._n_inputs, dtype=np.float64) - - def return_train_values(self, feat): - return self.rom.trainingSet.get(feat) - - def input_names(self): - return self._input_names + def __init__(self, **kwargs): + self._rom_file = kwargs.get("rom_file") + self._raven_framework = kwargs.get("raven_framework") + self._raven_framework = os.path.abspath(self._raven_framework) + if not os.path.exists(self._raven_framework): + raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') + if os.path.dirname(self._raven_framework).endswith("framework"): + sys.path.append(self._raven_framework) + else: + sys.path.append(os.path.join(self._raven_framework,"framework")) + import Driver + # de-serialize the ROM + self._rom_file = os.path.abspath(self._rom_file) + if not os.path.exists(self._rom_file): + raise IOError('The serialized (binary) file has not been found in location "' + str(self._rom_file)+'" !') + self.rom = pickle.load(open(self._rom_file, mode='rb')) + # + self.settings = self.rom.getInitParams() + # get input names + self._input_names = self.settings.get('Features') + # n_inputs + self._n_inputs = len(self._input_names) + # get output names + self._output_names = self.settings.get('Target') + # n_outputs + self._n_outputs = len(self._output_names) + # input values storage + self._input_values = np.zeros(self._n_inputs, dtype=np.float64) - def output_names(self): - return self._output_names + def return_train_values(self, feat): + return self.rom.trainingSet.get(feat) - def set_input_values(self, input_values): - assert len(input_values) == self._n_inputs - np.copyto(self._input_values, input_values) + def input_names(self): + return self._input_names - def evaluate_equality_constraints(self): - raise NotImplementedError('This method should not be called for this model.') + def output_names(self): + return self._output_names - def evaluate_outputs(self): - request = {k,np.asarray(v) for k,v in zip(self._input_names,self._input_values)} - outs = self.rom.evaluate(request) - return np.asaarray([outs[k].flatten() for k in self._output_names], dtype=np.float6) + def set_input_values(self, input_values): + assert len(input_values) == self._n_inputs + np.copyto(self._input_values, input_values) - def evaluate_jacobian_outputs(self): - request = {k,np.asarray(v) for k,v in zip(self._input_names,self._input_values)} - derivatives = self.rom.derivatives(request) - jac = np.zeros((self._n_outputs, self._n_inputs)) - for tc, target in enumerate(self._output_names): - for fc, feature in enumerate(self._input_names): - jac[tc,fc] = derivatives['d{}|d{}'.format(feature,target)] - return jac + def evaluate_equality_constraints(self): + raise NotImplementedError('This method should not be called for this model.') - def pyomoModel(ex_model): - m = pyo.ConcreteModel() - m.egb = ExternalGreyBoxBlock() - m.egb.set_external_model(ex_model) - for inp in ex_model.input_names(): - m.egb.inputs[inp].value = np.mean(ex_model.return_train_values(inp)) - m.egb.inputs[inp].setlb(np.min(ex_model.return_train_values(inp))) - m.egb.inputs[inp].setub(np.max(ex_model.return_train_values(inp))) - for out in ex_model.output_names(): - m.egb.outputs[out].value = np.mean(ex_model.return_train_values(out)) - m.egb.outputs[out].setlb(np.min(ex_model.return_train_values(out))) - m.egb.outputs[out].setub(np.max(ex_model.return_train_values(out))) - m.obj = pyo.Objective(expr=m.egb.outputs[out]) + def evaluate_outputs(self): + request = {k,np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + outs = self.rom.evaluate(request) + return np.asaarray([outs[k].flatten() for k in self._output_names], dtype=np.float6) - return m + def evaluate_jacobian_outputs(self): + request = {k,np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + derivatives = self.rom.derivatives(request) + jac = np.zeros((self._n_outputs, self._n_inputs)) + for tc, target in enumerate(self._output_names): + for fc, feature in enumerate(self._input_names): + jac[tc,fc] = derivatives['d{}|d{}'.format(feature,target)] + return jac - if __name__ == '__main__': - for cnt, item in enumerate(sys.argv): - if item.lower("-r"): - rom_file = sys.argv[cnt+1] - if item.lower("-f"): - raven_framework = sys.argv[cnt+1] - ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework}) - concreteModel = pyomoModel(ext_model) - ### here you should implement the optimization problem - ### +def pyomoModel(ex_model): + m = pyo.ConcreteModel() + m.egb = ExternalGreyBoxBlock() + m.egb.set_external_model(ex_model) + for inp in ex_model.input_names(): + m.egb.inputs[inp].value = np.mean(ex_model.return_train_values(inp)) + m.egb.inputs[inp].setlb(np.min(ex_model.return_train_values(inp))) + m.egb.inputs[inp].setub(np.max(ex_model.return_train_values(inp))) + for out in ex_model.output_names(): + m.egb.outputs[out].value = np.mean(ex_model.return_train_values(out)) + m.egb.outputs[out].setlb(np.min(ex_model.return_train_values(out))) + m.egb.outputs[out].setub(np.max(ex_model.return_train_values(out))) + m.obj = pyo.Objective(expr=m.egb.outputs[out]) + return m - """ +if __name__ == '__main__': + for cnt, item in enumerate(sys.argv): + if item.lower("-r"): + rom_file = sys.argv[cnt+1] + if item.lower("-f"): + raven_framework = sys.argv[cnt+1] + ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework}) + concreteModel = pyomoModel(ext_model) + ### here you should implement the optimization problem + ### +""" return template From 9dca4a0a0c228d4fa067054f2754d1f05b11ec96 Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Mon, 23 Aug 2021 15:14:52 -0600 Subject: [PATCH 10/57] Fixing various problems with grey model template. --- framework/Models/ROM.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 0af4202d2a..1df0efe5c5 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -1680,10 +1680,10 @@ def writePyomoGreyModel(self): # MODEL GENERATED BY RAVEN (raven.inl.gov) import pyomo.environ as pyo from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock -from pyomo.contrib.pynumero.dependencies import (numpy as np, scipy_sparse as spa) +from pyomo.contrib.pynumero.dependencies import (numpy as np) from pyomo.contrib.pynumero.asl import AmplInterface -from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP, ipopt, ipopt_available -import os, sys +from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP +import os, sys, pickle # RAVEN ROM PYOMO GREY MODEL CLASS class ravenROM(ExternalGreyBoxModel): @@ -1734,12 +1734,12 @@ def evaluate_equality_constraints(self): raise NotImplementedError('This method should not be called for this model.') def evaluate_outputs(self): - request = {k,np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} outs = self.rom.evaluate(request) return np.asaarray([outs[k].flatten() for k in self._output_names], dtype=np.float6) def evaluate_jacobian_outputs(self): - request = {k,np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} derivatives = self.rom.derivatives(request) jac = np.zeros((self._n_outputs, self._n_inputs)) for tc, target in enumerate(self._output_names): @@ -1765,9 +1765,9 @@ def pyomoModel(ex_model): if __name__ == '__main__': for cnt, item in enumerate(sys.argv): - if item.lower("-r"): + if item.lower() == "-r": rom_file = sys.argv[cnt+1] - if item.lower("-f"): + if item.lower() == "-f": raven_framework = sys.argv[cnt+1] ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework}) concreteModel = pyomoModel(ext_model) From 0a15a53b874506a6733ccc034c35d9c1a19b00da Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Mon, 23 Aug 2021 15:15:36 -0600 Subject: [PATCH 11/57] Output pickled rom as well. --- tests/framework/Models/External/serialize_pyomo.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/framework/Models/External/serialize_pyomo.xml b/tests/framework/Models/External/serialize_pyomo.xml index fd3e33605b..53c79d4670 100644 --- a/tests/framework/Models/External/serialize_pyomo.xml +++ b/tests/framework/Models/External/serialize_pyomo.xml @@ -5,11 +5,12 @@ SerializePyomo - sample, train, serialize + sample, train, serialize, pickle rom_out.py + rom_pickle.pk @@ -27,6 +28,10 @@ out_rom rom_out.py + + out_rom + rom_pickle.pk + From 220c502337da485675c0f1241d7afac3415d128c Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Thu, 26 Aug 2021 12:00:05 -0600 Subject: [PATCH 12/57] Additional work on making pyomo export work. --- framework/Models/ROM.py | 9 +++++++-- framework/utils/mathUtils.py | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 1df0efe5c5..b443c5c354 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -1736,7 +1736,8 @@ def evaluate_equality_constraints(self): def evaluate_outputs(self): request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} outs = self.rom.evaluate(request) - return np.asaarray([outs[k].flatten() for k in self._output_names], dtype=np.float6) + eval_outputs = np.asarray([outs[k].flatten() for k in self._output_names], dtype=np.float64) + return eval_outputs.flatten() def evaluate_jacobian_outputs(self): request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} @@ -1744,7 +1745,7 @@ def evaluate_jacobian_outputs(self): jac = np.zeros((self._n_outputs, self._n_inputs)) for tc, target in enumerate(self._output_names): for fc, feature in enumerate(self._input_names): - jac[tc,fc] = derivatives['d{}|d{}'.format(feature,target)] + jac[tc,fc] = derivatives['d{}|d{}'.format(target, feature)] return jac def pyomoModel(ex_model): @@ -1773,6 +1774,10 @@ def pyomoModel(ex_model): concreteModel = pyomoModel(ext_model) ### here you should implement the optimization problem ### + solver = pyo.SolverFactory('cyipopt') + solver.config.options['hessian_approximation'] = 'limited-memory' + results = solver.solve(concreteModel) + print(results) """ return template diff --git a/framework/utils/mathUtils.py b/framework/utils/mathUtils.py index 1f754cc241..4f1c0a17c1 100644 --- a/framework/utils/mathUtils.py +++ b/framework/utils/mathUtils.py @@ -27,7 +27,7 @@ import six from numpy import linalg -from utils.utils import UreturnPrintTag, UreturnPrintPostTag +from utils.utils import UreturnPrintTag, UreturnPrintPostTag, first from .graphStructure import graphObject import MessageHandler # makes sure getMessageHandler is defined @@ -996,7 +996,11 @@ def derivatives(f, x0, var = None, n = 1, h = None, target=None): """ assert(n <= 10) assert(isinstance(var, list) or isinstance(var, type(None))) + #XXX Possibly this should only be done sometimes + for key in x0: + x0[key] = np.atleast_1d(x0[key]) assert(len(list(x0.values())[0]) == 1) + #assert(first(x0.values()).size == 1) targets = [target] if target is None: checkWorking = f(x0) From 2d54d05d4d0e35f93b029d5bd440a6ba9ca7808b Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Tue, 31 Aug 2021 13:05:20 -0600 Subject: [PATCH 13/57] Adding description of test. --- .../Models/External/serialize_pyomo.xml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/framework/Models/External/serialize_pyomo.xml b/tests/framework/Models/External/serialize_pyomo.xml index 53c79d4670..9fcdd77771 100644 --- a/tests/framework/Models/External/serialize_pyomo.xml +++ b/tests/framework/Models/External/serialize_pyomo.xml @@ -1,6 +1,35 @@ + framework.Models.External.serialize_pyomo + cogljj + 2021-08-18 + Models.ROM + + This test serializes a ROM to use with PYOMO. + It can be used with: + cd tests/framework/Models/External + python3 SerializePyomo/rom_out.py -r SerializePyomo/rom_pickle.pk -f ../../../.. + + + This test uses the analytic model ''attenuate``, which is documented in the analytical test documentation. + Additionally, values of the ''from`` variables are exactly determined because they are set in the extmod methods: + \begin{itemize} + \item fromReadMoreXML: pi (3.14159) + \item fromInit : sqrt(pi) (1.77245) + \item fromCNISelf : pi/2 (1.57080) + \item fromCNIDict : 2*sqrt(pi) (3.54491) + \end{itemize} + The exit strength ''ans`` is analytic with results as follows (Note that for testing purposes, the model always adds + 0.05 to \texttt{y2} before calculating the exit strength): + \begin{itemize} + \item \texttt{y1}, \texttt{y2}, \texttt{ans} + \item 0.0, 0.0, 0.97531 + \item 0.0, 1.0, 0.59156 + \item 1.0, 0.0, 0.59156 + \item 1.0, 1.0, 0.35880 + \end{itemize} + From 5718fe5e51c0f2007ee93959ea13d3c63ab67589 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Tue, 19 Oct 2021 16:51:49 -0600 Subject: [PATCH 14/57] Add files via upload New test model file that replicates the example provided by Pyomo. --- .../Models/External/AllMethods/simple.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/framework/Models/External/AllMethods/simple.py diff --git a/tests/framework/Models/External/AllMethods/simple.py b/tests/framework/Models/External/AllMethods/simple.py new file mode 100644 index 0000000000..7dd092942b --- /dev/null +++ b/tests/framework/Models/External/AllMethods/simple.py @@ -0,0 +1,45 @@ +import numpy as np +import math + +def minimum_value(c1,c2,x1,x2): + """ + Calculates minimum value for function c1*x1+c2*x2 based on constraints. + @ In, c1, float, coefficeint of x1 + @ In, x1, float, function value 1 + @ In, c2, float, coefficient of x2 + @ In, x2, float, function value 2 + @ Out, float, minimum value + """ + return np.float(c1*x1+c2*x2) + +def constraint_value(a1,a2,x1,x2): + """ + Calculates canstraint value for function x1 and x2. + @ In, a1, float, coefficeint of x1 + @ In, x1, float, function value 1 + @ In, a2, float, coefficient of x2 + @ In, x2, float, function value 2 + @ Out, float, constraint value + """ + return a1*x1+a2*x2 + +def test_constraint(constraint,b): + """ + Tests if the selection of x1 and x2 satisfies the constraints set by "b". + @ In, constraint, float, constraint value + @ In, b, float, contraint parameter + @ Out, logic, decision for validity + """ + if constraint >= b: + test_condition = True + else: + test_condition = False + return test_condition + +def run(self,Input): # Input as dict + constraint = constraint_value(self.a1,self.a2,self.x1,self.x2) + test = test_constraint(constraint,self.b) + if test is True and self.x1 >= 0 and self.x2 >= 0: + self.ans = minimum_value(self.c1,self.c2,self.x1,self.x2) + else: + self.ans = np.float(10**6) From 23d61f8c450947bd6c29fa37aaa4f4186813c4b8 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Tue, 19 Oct 2021 16:53:32 -0600 Subject: [PATCH 15/57] Add files via upload Added xml file that will sample and train simple.py in AllMethods as done in serialize_pyomo.xml for attenuate.py. --- .../Models/External/simple_pyomo.xml | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 tests/framework/Models/External/simple_pyomo.xml diff --git a/tests/framework/Models/External/simple_pyomo.xml b/tests/framework/Models/External/simple_pyomo.xml new file mode 100644 index 0000000000..03ea34487f --- /dev/null +++ b/tests/framework/Models/External/simple_pyomo.xml @@ -0,0 +1,122 @@ + + + + framework.Models.External.serialize_pyomo + cogljj + 2021-08-18 + Models.ROM + + This test serializes a ROM to use with PYOMO. + It can be used with: + cd tests/framework/Models/External + python3 SerializePyomo/simple_out.py -r SerializePyomo/simple_pickle.pk -f ../../../.. + + + This test uses the example from Pyomo's manual for concrete and abstract models. + The following conditions are used to determine the minimum value of: + \begin{equation} + 2*x_{1}+3*x_{2} + \end{equation} + The conditions are the following: + \begin{equation} + 3*x_{1}+4*x_{2}\geq 1,\qquad x_{1}\geq 0,\qquad x_{2}\geq 0 + \end{equation} + The GNU Linear Programming Kit (glpk) option is used in Pyomo to solve the problem. + The results are the following: + \begin{itemize} + \item Lower bound: 0.6666666666666667 + \item Upper bound: 0.6666666666666667 + \item Number of objectives: 1 + \item Number of constraints: 2 + \item Number of variables: 3 + \item of nonzeros: 3 + \item Sense: minimize + \item number of solutions: 1 + \item Status: feasible + \item $x_{1}$ value: 0.333333333333333 + \item $x_{2}$ value: 0.000000000000000 + \item $2*x_{1}+3*x_{2}$ value: 0.666666666666666 + \end{itemize} + + + + + SerializePyomo + sample, train, serialize, pickle + + + + simple_out.py + simple_pickle.pk + + + + + dummyIN + simple + grid + samples + + + samples + simple_rom + + + simple_rom + simple_out.py + + + simple_rom + simple_pickle.pk + + + + + + x1,x2,a1,a2,b,c1,c2 + ans + + + linear_model|LinearRegression + x1,x2 + ans + + + + + + 0 + 1 + + + + + + + dist + 0 1 + + + dist + 0 1 + + 3.0 + 4.0 + 1.0 + 2.0 + 3.0 + + + + + + x1,x2 + + + x1, x2 + ans + + + + + From a8ee07f9617f3dbc630518dc6915d71ccd31168a Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Tue, 19 Oct 2021 16:57:30 -0600 Subject: [PATCH 16/57] Update rom.tex This adds the user manual description for serialization and file generation to run RAVEN model in Pyomo. --- doc/user_manual/rom.tex | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index 8fd01293d1..6abf64bf57 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -3472,3 +3472,57 @@ \subsubsection{DMD} %%%%% ROM Model - TensorFlow-Keras Interface %%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \input{kerasROM.tex} + +\subsubsection{SerializePyomo} +\label{subsubsec:serializepyomo} +For the purpose of exporting RAVEN trained ROMs in Pyomo format, the platform to print the Pyomo python file is estblished. +The capabilities implemented enable to retrieve sampled and trained ROMs, pickle (serialize), and output python files as +defined in the \xmlNode{IOStep}. + +The trained file is copied and processed separately to output two files required to run +Pyomo. The python output file contains the infrastructure inheriting the External Grey Box Model and others included in +the Pyomo contributions package Pynumero to print a file with Pyomo syntax. The `.pk' output file is the pickled version of the +trained ROM and can be ran with the python output file. + +Since the code developed in ROMs is not a subtype, the code activation is triggered by file type. To ensure developed Pyomo code +generator is utilized in ROMs, one of the file extensions in the output of the \xmlNode{IOStep} must be `.py'. The subtype of the +ROM model can be of any kind and is the user's decision to choose the training method. + +As noted in pickledRom (\ref{subsubsec:pickledROM}), when loading ROMs from file, the user is expected to know the I/O of a ROM +and loading of the ROM file must use \xmlNode{IOStep}. + +\textbf{Example:} For this example the external model is trained and further loaded as `out_rom'. The loaded trained model is separately +processed to `rom_out.py' and `rom_pickle.pk'. +\begin{lstlisting}[style=XML,morekeywords={name,subType}] + + ... + + rom_out.py + rom_pickle.pk + + ... + + ... + + ... + attenuate + ... + samples + + + samples + out_rom + + + out_rom + out_rom.py + + + out_rom + out_pickle.pk + + ... + + ... + + \end{lstlisting} From 78b16dc6bccf883c0364421f508340f36c98e6f4 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Tue, 19 Oct 2021 17:07:17 -0600 Subject: [PATCH 17/57] Update rom.tex Fixed indenting. --- doc/user_manual/rom.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index 6abf64bf57..33f3ea2caf 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -3525,4 +3525,4 @@ \subsubsection{SerializePyomo} ... - \end{lstlisting} +\end{lstlisting} From fc05c93dcce6d8f07a6627f474afcef9f290654c Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 20 Oct 2021 15:28:23 -0600 Subject: [PATCH 18/57] Added simply.py in tests file --- tests/framework/Models/External/tests | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index d95eae2284..0444d3f481 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -14,6 +14,11 @@ input = 'serialize_pyomo.xml' output = 'SerializePyomo/rom_out.py' [../] + [./pyomo_basic] + type = 'RavenFramework' + input = 'simple_pyomo.xml' + output = 'SerializePyomo/simple_out.py' + [../] [] From 7affcf49bdfe888fe166a5558237dba6d9d8da84 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Mon, 25 Oct 2021 12:19:20 -0600 Subject: [PATCH 19/57] Update rom.tex Changing underscores in-text to correct syntax. --- doc/user_manual/rom.tex | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index a870baf064..9f39baee4e 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -207,11 +207,10 @@ \subsubsection{SerializePyomo} generator is utilized in ROMs, one of the file extensions in the output of the \xmlNode{IOStep} must be `.py'. The subtype of the ROM model can be of any kind and is the user's decision to choose the training method. -As noted in pickledRom (\ref{subsubsec:pickledROM}), when loading ROMs from file, the user is expected to know the I/O of a ROM -and loading of the ROM file must use \xmlNode{IOStep}. +When loading ROMs from file, the user is expected to know the I/O of a ROM and loading of the ROM file must use \xmlNode{IOStep}. -\textbf{Example:} For this example the external model is trained and further loaded as `out_rom'. The loaded trained model is separately -processed to `rom_out.py' and `rom_pickle.pk'. +\textbf{Example:} For this example the external model is trained and further loaded as `out\_rom'. The loaded trained model is separately +processed to `rom\_out.py' and `rom\_pickle.pk'. \begin{lstlisting}[style=XML,morekeywords={name,subType}] ... From 905cb5fb2ade37f6387a043c6be78db5c51128ad Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Wed, 27 Oct 2021 10:39:28 -0600 Subject: [PATCH 20/57] Update ROM.py When the LearningGate.py was merged with ROM.py, the derivative function in the LearningGate.py was not added. The derivative function in the ROM.py was originally using the derivative function from the LearningGate,py which was deprecated. The derivative function in ROM.py now directly uses the mathUtils as the LearningGate.py did. Since the self.supervisedEngine is now an artifact of past ROM.py versions, it has been deleted in this commit. --- framework/Models/ROM.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 4d4e9e2ed5..0e4594799d 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -108,7 +108,6 @@ def __init__(self): """ super().__init__() self.amITrained = False # boolean flag, is the ROM trained? - self.supervisedEngine = None # dict of ROM instances (== number of targets => keys are the targets) self.printTag = 'ROM MODEL' # label self.cvInstanceName = None # the name of Cross Validation instance self.cvInstance = None # Instance of provided cross validation @@ -429,10 +428,21 @@ def derivatives(self, request, feats = None, order=1): 'd(feature1)/d(target1)':np.array(...)}. For example, {"x1/y1":np.array(...),"x2/y1":np.array(...),etc.} """ - inputToROM = self._inputToInternal(request) - derivatives = self.supervisedEngine.derivatives(inputToROM, order=order, feats=feats) - for k,v in derivatives.items(): - derivatives[k] = np.atleast_1d(v) + if self.pickled: + self.raiseAnError(RuntimeError,'ROM "', self.name, '" has not been loaded yet! Use an IOStep to load it.') + if not self.amITrained: + self.raiseAnError(RuntimeError, "ROM ", self.name, " has not been trained yet and, consequentially, can not be evaluated!") + derivatives = {} + if self.segment: + derivatives = mathUtils.derivatives(self.supervisedContainer[0].evaluate, request, var=feats, n=order) + else: + for rom in self.supervisedContainer: + sliceEvaluation = mathUtils.derivatives(rom.evaluate, request, var=feats, n=order) + if len(list(derivatives.keys())) == 0: + derivatives.update(sliceEvaluation) + else: + for key in derivatives.keys(): + derivatives[key] = np.append(derivatives[key],sliceEvaluation[key]) return derivatives def _externalRun(self,inRun): From 977fa226f2e1d9acf72c00affbb65befd2dc7a79 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Mon, 29 Nov 2021 15:19:45 -0700 Subject: [PATCH 21/57] Update tests Removed simple test. --- tests/framework/Models/External/tests | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index 0444d3f481..d95eae2284 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -14,11 +14,6 @@ input = 'serialize_pyomo.xml' output = 'SerializePyomo/rom_out.py' [../] - [./pyomo_basic] - type = 'RavenFramework' - input = 'simple_pyomo.xml' - output = 'SerializePyomo/simple_out.py' - [../] [] From 247857d05c83bad88b12d1a6834b77e5dec1a0e6 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Mon, 29 Nov 2021 15:20:18 -0700 Subject: [PATCH 22/57] Delete simple_pyomo.xml Removing simple test --- .../Models/External/simple_pyomo.xml | 121 ------------------ 1 file changed, 121 deletions(-) delete mode 100644 tests/framework/Models/External/simple_pyomo.xml diff --git a/tests/framework/Models/External/simple_pyomo.xml b/tests/framework/Models/External/simple_pyomo.xml deleted file mode 100644 index fb02b56d14..0000000000 --- a/tests/framework/Models/External/simple_pyomo.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - - framework.Models.External.serialize_pyomo - cogljj - 2021-08-18 - Models.ROM - - This test serializes a ROM to use with PYOMO. - It can be used with: - cd tests/framework/Models/External - python3 SerializePyomo/simple_out.py -r SerializePyomo/simple_pickle.pk -f ../../../.. - - - This test uses the example from Pyomo's manual for concrete and abstract models. - The following conditions are used to determine the minimum value of: - \begin{equation} - 2*x_{1}+3*x_{2} - \end{equation} - The conditions are the following: - \begin{equation} - 3*x_{1}+4*x_{2}\geq 1,\qquad x_{1}\geq 0,\qquad x_{2}\geq 0 - \end{equation} - The GNU Linear Programming Kit (glpk) option is used in Pyomo to solve the problem. - The results are the following: - \begin{itemize} - \item Lower bound: 0.6666666666666667 - \item Upper bound: 0.6666666666666667 - \item Number of objectives: 1 - \item Number of constraints: 2 - \item Number of variables: 3 - \item of nonzeros: 3 - \item Sense: minimize - \item number of solutions: 1 - \item Status: feasible - \item $x_{1}$ value: 0.333333333333333 - \item $x_{2}$ value: 0.000000000000000 - \item $2*x_{1}+3*x_{2}$ value: 0.666666666666666 - \end{itemize} - - - - - SerializePyomo - sample, train, serialize, pickle - - - - simple_out.py - simple_pickle.pk - - - - - dummyIN - simple - grid - samples - - - samples - simple_rom - - - simple_rom - simple_out.py - - - simple_rom - simple_pickle.pk - - - - - - x1,x2,a1,a2,b,c1,c2 - ans - - - x1,x2 - ans - - - - - - 0 - 1 - - - - - - - dist - 0 1 - - - dist - 0 1 - - 3.0 - 4.0 - 1.0 - 2.0 - 3.0 - - - - - - x1,x2 - - - x1, x2 - ans - - - - - From b734c1cf4e36674f79b984534592f5bfdf52f100 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Mon, 29 Nov 2021 15:20:44 -0700 Subject: [PATCH 23/57] Delete simple.py Removing simple test --- .../Models/External/AllMethods/simple.py | 45 ------------------- 1 file changed, 45 deletions(-) delete mode 100644 tests/framework/Models/External/AllMethods/simple.py diff --git a/tests/framework/Models/External/AllMethods/simple.py b/tests/framework/Models/External/AllMethods/simple.py deleted file mode 100644 index 7dd092942b..0000000000 --- a/tests/framework/Models/External/AllMethods/simple.py +++ /dev/null @@ -1,45 +0,0 @@ -import numpy as np -import math - -def minimum_value(c1,c2,x1,x2): - """ - Calculates minimum value for function c1*x1+c2*x2 based on constraints. - @ In, c1, float, coefficeint of x1 - @ In, x1, float, function value 1 - @ In, c2, float, coefficient of x2 - @ In, x2, float, function value 2 - @ Out, float, minimum value - """ - return np.float(c1*x1+c2*x2) - -def constraint_value(a1,a2,x1,x2): - """ - Calculates canstraint value for function x1 and x2. - @ In, a1, float, coefficeint of x1 - @ In, x1, float, function value 1 - @ In, a2, float, coefficient of x2 - @ In, x2, float, function value 2 - @ Out, float, constraint value - """ - return a1*x1+a2*x2 - -def test_constraint(constraint,b): - """ - Tests if the selection of x1 and x2 satisfies the constraints set by "b". - @ In, constraint, float, constraint value - @ In, b, float, contraint parameter - @ Out, logic, decision for validity - """ - if constraint >= b: - test_condition = True - else: - test_condition = False - return test_condition - -def run(self,Input): # Input as dict - constraint = constraint_value(self.a1,self.a2,self.x1,self.x2) - test = test_constraint(constraint,self.b) - if test is True and self.x1 >= 0 and self.x2 >= 0: - self.ans = minimum_value(self.c1,self.c2,self.x1,self.x2) - else: - self.ans = np.float(10**6) From 180cefa272b3bf6093ee9d2ab6e0837371292845 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Tue, 14 Dec 2021 10:31:25 -0700 Subject: [PATCH 24/57] Updated the trigger to initiate Pyomo GreyBoxModel print in ROM.py --- doc/user_manual/rom.tex | 2 +- framework/BaseClasses/BaseEntity.py | 1 - framework/Files.py | 13 +++++++++++++ framework/Models/Model.py | 17 ++++++++--------- framework/Steps/IOStep.py | 3 ++- .../Models/External/serialize_pyomo.xml | 2 +- 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index 9f39baee4e..22d8d5a20e 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -194,7 +194,7 @@ \subsection{ROM} \subsubsection{SerializePyomo} \label{subsubsec:serializepyomo} -For the purpose of exporting RAVEN trained ROMs in Pyomo format, the platform to print the Pyomo python file is estblished. +For the purpose of exporting RAVEN trained ROMs in Pyomo format, the platform to print the Pyomo python file is established. The capabilities implemented enable to retrieve sampled and trained ROMs, pickle (serialize), and output python files as defined in the \xmlNode{IOStep}. diff --git a/framework/BaseClasses/BaseEntity.py b/framework/BaseClasses/BaseEntity.py index 67a73b34d2..8a8f51ade1 100644 --- a/framework/BaseClasses/BaseEntity.py +++ b/framework/BaseClasses/BaseEntity.py @@ -245,7 +245,6 @@ def _validateSolutionExportVariables(self, solutionExport): # get acceptable names fromSolnExport = set(self.getSolutionExportVariableNames()) acceptable = set(self._formatSolutionExportVariableNames(fromSolnExport)) - #solutionExport.addExpectedMeta(fromSolnExport) # remove registered solution export names first remaining = requested - acceptable # anything remaining is unknown! diff --git a/framework/Files.py b/framework/Files.py index a96f288f67..8d119d260e 100644 --- a/framework/Files.py +++ b/framework/Files.py @@ -166,6 +166,14 @@ def getLinkedCode(self): """ return self.__linkedModel + def getModelType(self): + """ + Retriever for model type. + @ In, None + @ Out, getLinkedCode, string, string path + """ + return self.modelType + # setting tools # def setPath(self,path): """ @@ -489,6 +497,8 @@ def initialize(self, filename, path='.', type='internal'): self.perturbed = False self.name = filename + print("I am Here! Files RAVENGenerated") + # # # @@ -531,6 +541,7 @@ def _readMoreXML(self,node): self.subDirectory = node.attrib.get('subDirectory',"") self.setAbsFile(os.path.join(self.subDirectory,node.text.strip())) self.alias = node.attrib.get('name' ,self.getFilename()) + self.modelType = node.attrib.get('modelType' ,"") def __getstate__(self): """ @@ -542,6 +553,7 @@ def __getstate__(self): stateDict['perturbed' ] = self.perturbed stateDict['subDirectory'] = self.subDirectory stateDict['alias' ] = self.alias + stateDict['modelType' ] = self.modelType return stateDict def __setstate__(self,stateDict): @@ -554,6 +566,7 @@ def __setstate__(self,stateDict): self.perturbed = stateDict['perturbed' ] self.subDirectory = stateDict['subDirectory'] self.alias = stateDict['alias' ] + self.modelType = stateDict['modelType' ] # diff --git a/framework/Models/Model.py b/framework/Models/Model.py index 86da8c36ae..9262505977 100644 --- a/framework/Models/Model.py +++ b/framework/Models/Model.py @@ -381,11 +381,10 @@ def initialize(self,runInfo,inputs,initDict=None): @ In, runInfo, dict, it is the run info from the jobHandler @ In, inputs, list, it is a list containing whatever is passed with an input role in the step @ In, initDict, dict, optional, dictionary of all objects available in the step is using this model - @ In, None """ pass - def serialize(self,fileo,**kwargs): + def serialize(self,fileObjIn,**kwargs): """ This method is the base class method that is aimed to serialize the model (and derived) instances. @ In, fileo, str or File object, the filename of the output serialized binary file or the RAVEN File instance @@ -393,14 +392,14 @@ def serialize(self,fileo,**kwargs): @ Out, None """ import cloudpickle - if isinstance(fileo,str): - fileobj = open(filename, mode='wb+') + if isinstance(fileObjIn,str): + fileObj = open(filename, mode='wb+') else: - fileobj = fileo - fileobj.open(mode='wb+') - cloudpickle.dump(self,fileobj, protocol=pickle.HIGHEST_PROTOCOL) - fileobj.flush() - fileobj.close() + fileObj = fileObjIn + fileObj.open(mode='wb+') + cloudpickle.dump(self,fileObj, protocol=pickle.HIGHEST_PROTOCOL) + fileObj.flush() + fileObj.close() @abc.abstractmethod def createNewInput(self,myInput,samplerType,**kwargs): diff --git a/framework/Steps/IOStep.py b/framework/Steps/IOStep.py index 842fd42ef5..071f4da340 100644 --- a/framework/Steps/IOStep.py +++ b/framework/Steps/IOStep.py @@ -123,7 +123,8 @@ def _localInitializeStep(self,inDictionary): if isinstance(outputs[i],Files.File): #XXX switching on the file extension is not very # extendable. Possibly should have a type or something - if 'py' == outputs[i].getExt().lower(): + #Past condition: if 'py' == outputs[i].getExt().lower(): + if 'Pyomo' == outputs[i].getModelType(): self.actionType.append('MODEL-PYOMO') else: self.actionType.append('MODEL-FILES') diff --git a/tests/framework/Models/External/serialize_pyomo.xml b/tests/framework/Models/External/serialize_pyomo.xml index 833b734d5f..2ddf6a9f2b 100644 --- a/tests/framework/Models/External/serialize_pyomo.xml +++ b/tests/framework/Models/External/serialize_pyomo.xml @@ -38,7 +38,7 @@ - rom_out.py + rom_out.py rom_pickle.pk From 8cb69f457197257bbd524455f2202f574f58dfa2 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Wed, 22 Dec 2021 10:01:22 -0700 Subject: [PATCH 25/57] Update Files.py Forgot to remove placeholder comment --- framework/Files.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/framework/Files.py b/framework/Files.py index 8d119d260e..97b7ecf489 100644 --- a/framework/Files.py +++ b/framework/Files.py @@ -497,8 +497,6 @@ def initialize(self, filename, path='.', type='internal'): self.perturbed = False self.name = filename - print("I am Here! Files RAVENGenerated") - # # # From 3df2385345a07fa9a94c812d0c6aafd9dc8105d4 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Thu, 13 Jan 2022 11:43:52 -0700 Subject: [PATCH 26/57] Uploading new rom.py to git pull repository --- framework/Models/ROM.py | 1 - 1 file changed, 1 deletion(-) diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 37f3dce425..621f32d63d 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -599,7 +599,6 @@ def writeXML(self, what='all'): def writePyomoGreyModel(self): """ Called by the OutStreamPrint object to cause the ROM to print itself - @ In, what, string, optional, keyword requesting what should be printed @ Out, xml, xmlUtils.StaticXmlElement, written meta """ template = r""" From ebe1aefcdd128505ab7041b2c3cdb939f315d262 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Mon, 17 Jan 2022 13:16:41 -0700 Subject: [PATCH 27/57] Added dependencies and pynumero extension to raven libraries --- dependencies.xml | 3 +++ scripts/establish_conda_env.sh | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/dependencies.xml b/dependencies.xml index 735d1169b9..4ae064adbc 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -60,6 +60,9 @@ Note all install methods after "main" take 1.1 0.9.39 + + + remove diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index 54848fa39f..eff5240f35 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -109,6 +109,15 @@ function install_libraries() local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset forge)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... conda-forge command: ${COMMAND}; fi ${COMMAND} + if [[ ${COMMAND} == *"pyomo"* ]]; + then + activate_env + #command -v pyomo + local PYNUMEROEXT=`echo pyomo download-extensions` + local PYNUMEROBLD=`echo pyomo build-extensions` + ${PYNUMEROEXT} + ${PYNUMEROBLD} || echo "Pyomo build failed" + fi # pip only activate_env if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from PIP-ONLY ...; fi @@ -139,6 +148,15 @@ function create_libraries() local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action create --subset forge)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... conda-forge command: ${COMMAND}; fi ${COMMAND} + if [[ ${COMMAND} == *"pyomo"* ]]; + then + activate_env + #command -v pyomo + local PYNUMEROEXT=`echo pyomo download-extensions` + local PYNUMEROBLD=`echo pyomo build-extensions` + ${PYNUMEROEXT} + ${PYNUMEROBLD} || echo "Pyomo build failed" + fi # pip only activate_env if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from PIP-ONLY ...; fi From 37b6517ef5ec7f79e6a75f443c4924ff50d1011e Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Tue, 18 Jan 2022 15:47:31 -0700 Subject: [PATCH 28/57] Updated Dependencies --- dependencies.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dependencies.xml b/dependencies.xml index 4ae064adbc..a89426503d 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -62,7 +62,9 @@ Note all install methods after "main" take + + remove From ee72b0591dcefbb82cecaf432f379aabbae4b3f4 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 26 Jan 2022 16:35:33 -0700 Subject: [PATCH 29/57] Updated the pyomo in ROM.py to enable terminal printing. --- framework/Models/ROM.py | 9 ++ .../gold/SerializePyomo/GrayModelOutput.txt | Bin 0 -> 2476 bytes .../External/gold/SerializePyomo/rom_out.py | 111 ++++++++++++++++++ .../gold/SerializePyomo/rom_pickle.pk | Bin 0 -> 8314 bytes 4 files changed, 120 insertions(+) create mode 100644 tests/framework/Models/External/gold/SerializePyomo/GrayModelOutput.txt create mode 100644 tests/framework/Models/External/gold/SerializePyomo/rom_out.py create mode 100644 tests/framework/Models/External/gold/SerializePyomo/rom_pickle.pk diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 621f32d63d..11d069de0f 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -703,6 +703,15 @@ def pyomoModel(ex_model): solver.config.options['hessian_approximation'] = 'limited-memory' results = solver.solve(concreteModel) print(results) + count = 0 + with open("GrayModelOutput.txt", "r") as f: + lines = f.readlines() + with open("GrayModelOutput.txt", "w") as f: + for line in lines: + if line == "Problem:": + count += 1 + if count == 1: + f.write(line) """ return template diff --git a/tests/framework/Models/External/gold/SerializePyomo/GrayModelOutput.txt b/tests/framework/Models/External/gold/SerializePyomo/GrayModelOutput.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6fab02c3cc39557e9b21e0288154cb3ec64cad6 GIT binary patch literal 2476 zcmeHDO;5r=5cS+&F*h_3LWLr5^RC2*#`AVNEhFtrvmcQ9>zxJ@A;z;q*v4$~msc&y> z`|~!P%(CVEYBA5Ud77=VbfJt*Y;vVY36g)H4Rk=ihV%ll!ag`d9DYFu~Zg-VjqmpO|c=KA=16#CFt58+vLmA!;_+ z>+pghItD9+{_fK}q4%cJAs;=|l{gB!1_C-&HUSIp(ORXHv93EX0Swe)O+TnQHKvAE zbdOu`CXg>X;tIkJ)qyoog=`Um)aZ=fJX~ADkR$jOEtRAO`AJFZy_t+O(7TFT_1J^T j_aP@~c9$-*)na-&=~7v*-w8?jWw@ZAH{J1+DM`Kn(xs@v7%k)u}$! z(_Pi6%d9)ggrHz&COX0^Z6Si4kjFeA5TFPNO9*(&1Mp&mc;E>nBpxtEF$Bp<;QP<1 zuIb)2-f_f8Q(d?7pa1fm@BjZ(Upx8Rfia=}wYF z_IMPrHh-S^PM3$NT)aMhLrmrG&PFi{JtSJp3HyR?zY*l)*fD+mh(~4 z?}yOfQW!?*ft=OSY07;$?fI)gHwbZmh6Jy|DrLk}?Gadu44w{~1k zI^0Uy>Dq}z&e}BmqG!i6Ogty22E5x1?xu1&a|6zNp2Q)8s?&E_=zBg4o9@9we#}xi z$-@u|$P*!V(nL-vfsvB-DS7e}Un=%e@mV?fi7!=3J-poxIrs64>oT5vQk_-s4809*@#*XGt6X4MClzkFS}R`5?FgDO)RmnE>FgTK4hu5%9B2WujF)) z#H}!Zz$F$%yl27ZH>?h~V^UT=)e1Qr1tV6Xo~0i^-ts>F<=ko)V?5U*I80sVap>8p zJihu8;>ja~pT%$ir&muWu?JjT_4`SDk&%ODUPzc9Q@|e}9Bap3PhU$nQn8Bj*O3ng^#`PN)FI|I{?QUSP?n;cwtR&`2%hRnLKe+4XH%UWL6$@N0?y}HhFzQMe zBz^Vre2>S>feveY5T$Pb$io{qS;%_ook`a97@aN4Nsn|UUrfPO`ZSUAEwxfwEy!2z ziCcG4@iJV0f+eq#$K`aCSX;{BOV>P~vv7kW%J5f!u+4;t|3bnMS`1mTL(XD;D+qUh zqKh8kcAhysFOIpRyhSdYxfx*nYyz`#?s+eQYX|CLn%Q=}wn}b!bXzyP2 zu@ZfouT!ko<=o*NTrIZpAPt`%-of*Q7Utuj%WSUr;8!-2J`Z=jh&$I{4k5mw`fJc1&iYC^Hi7OZ0)2FYe;y1-_C2}tD zy)P&HCDx7L>zRmyFa5U0+!J|VS}i;14_dbo&Syt%x?Zep$5IA8++Npl22c!s~c*WW!MSAJOzmh;#&|RPvv$+ zEyPg#j?Pfx&3*CKf%x`5y)$VC+-;@uXZ)ns9~A5$M<+*26$_rvc^{>|;X z7t+@;vz!c>ztc*Qpk^Z=B*A;FRQ!SXBhpX&$*6X3V>XSl_bv>7)ogQWB@LF6yPFxD3n6i&nBbiY+kyDWvm^TUG-kP~bA8*v3rGi;V1I z4g$xTosM|BM_K$p&LL`B0_^ETl*vy<9NGXOt?U)zExPOzv#aEj%4h6ukT_cP!pM0l zpKkTo4hM#Lm&bwMvMIeGCg5Hx@!%71MV3Sn{%Hu`>~lL6D{}GBa+NI}Aen)IjB=x7 zS+-qtnvIHCE!J4Eh~H|ZQZb8F&dcd~Od*o}@1Zk`dkXi7xmOTEJxy>-9mCZRNXBMVK+1{K%7PN8R;ep-e)NMRuwZ zg^Z7seK8auxY9a`3=|PmjGZ%}k-=XjW%xOaI`W*b%VL(-rO?=bgJ8Q}5Pm#I!E8`M zWGsWI?En=pDN0xxdO$Bx zy2OVjUL5f*v`VFd`J!lRE~WnefM&<*;q7d_IEP4#*-9YJ28Mr{iWMxYUV2sOH%6|!((sAk9k zVk1>>omc7m%$cvgVtX;ahbn9>tHSni65Kss;nk411GwU;L+BeGz(}cetzggSs116V&FleGLjMZNgEk3?@r;z*T{7xabTE zV08#!06Ql01CpjL60x9^692p`{zWe2&8p`A59ItMB%&{%e#%tS>|f{AYSk<@DowX( zR*DtNF-_YxYvl?rm8%vq6|ukgkIvk&i3yVbLs`vWL)sCKbI2t#9#7*7avEV$^N%Q~ zuNQsf#VZ^+6^xC_c3;kr;A`LnB-CZ`v*H=?tay$mdnh?x-zbWo9KfN~i|`#d*vQSL z^_}&<5%|7GOVUO22$bZ$O*<2?rgIF3L@@2OAB&$*gw94tm!7NBP>hh0ynH#|^}1cM z6R2}Wg(OJp2?|B#)64g})Bsx_xxD!PtY zvnvg&Vsq1}nvA;@r(Dbf@tI*D#QVsMFNz}W ze;NZG8|TlD8Xzj#3mQ90I|_@^urrAnwUSoNvC8FI+2NI%*<@D3WmVI(YPCw;teMqD zuH^ZllAU>+iK^5Vt5A0qa0|T=wXbwT-#m?*`<*2mDcZNh%OjCSU^>5uCnI4?I?vFJ zpThxel`}2)KWnlm2xtnij|1VU`0)@i>)QA8!|rFy{gsAkx@Hm1uA63!vqsgiOHI>p z>#S^XUe7JNfEmeGKP30dnW3;0*!eS!9S{7g)&`b$o`u}3J=5t2NE{=iSA5IZ5=bGV zX_XRF8~rrWx>{VsBq%BXcFMCf!>}~Sg$61S=d`|zTGQCnaQx~kA$N%#BW`~VQu-S~ zPxs;B-r01BqA{;_s>;?*9Y5j&xu}_qR;2RmgY+TK&TXllb#{4>^MZv4LKaB?zP}E#h)nQ5Z7?_<1nS4P-gSH?3Q1fR# z&wTudUvuw}f60rNy*72=B;+~=;Ty}Bkvh+6!l0!ONh;xD(Nie0n(r{3{69r6S4>CI z7>%CsQPC?rK75@k@Zv+^dUCuU8Vj3QX%tJvvRN!PZL?gjF^k)cdeJR6%cV-uuI6F8 zGZwbRagnqrzIK%gZEDAmsruY^s7T4Ynj`w0fszra$rxWjN?PugA-x&PywENvk>?B^ z+h-LWQ-MIy)S2>d^QoX|PQ3GYh(F8}Q>!nHIi+qD%awB7Vin#nxsA4|#v2W%#G95= zZ90b>u`}uv@p*XRn^@gOjFQQqLW-IQRhh4ifKhgzd)XzpD zNV;e%5DcUV=tSc6(!ZyAti2CQ9NFk!`l!+4x{pd3XY(=Mc{nH^k~+;siJ7d*co7s{ zFBVzJbu7ke)pEJXo6W=2xi=P<7eOw+j5~*zQz?nqHFqHM2fFvc_AG@g@9f6V20GT; zL%=U~A`2PkC~+Vbs#CL~0fqD6u0W+egzIQ|H_|bIt5M*>`2^`9@S}opRYSuHoWVGP z=si^wA6i4VMlm!L1DVvB;HmEQ0Kz=YKtyi)vXf9HOWg~KeQvAf_v{FMI=_(lGp{v1 zr%PKU94&l;?m#)=9CV`{Zah%^1H7f)8L@s84o9s8N3w3P zdfBd(a*{e2#yzV>Iuv$QhIEQ__L$m3eww4rLp-*(6}zAlN5r?cb_$C(c@7@wzX-g4S zc%?lx<8WbSTa!p;u)+^wpvpBaV~>}g%VYmh`yk`bXclk_Q+qB^l+cz0a0|S%E^T_D zF9hQQ@lNKZ{l^pW2Xf{ML7*FjN2-j4e1~`3V-40|MTlP!olBgJzl)pyFgZ6w93*{d=jcnVe$H6imr>D=u=y8(EsJ58z(}z-TS9cI1>v2jJFPF?(%{0qJvr(;C$fmAS;oRg-!yYNJ zzdsrye3i|FQ9aQFy!~NnQcfqr# zz5kQ=5P-dZL+|-Ne0=Cz0DS{Hs)*sxp~&|xsLu%}7&`F(TgE@mz5kg%um1am?`-_& Z>yz*PQ~rDJkE_vZ|MmAuMv_>C{{{bzaTEXm literal 0 HcmV?d00001 From 313f69a47393d3945d91400f1d11ca2dbdbf1d5b Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Thu, 3 Feb 2022 11:26:27 -0700 Subject: [PATCH 30/57] Switching to getType instead of getModelType This matches the upcoming FMU/FMI pull request. --- framework/Files.py | 11 ----------- framework/Steps/IOStep.py | 2 +- tests/framework/Models/External/serialize_pyomo.xml | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/framework/Files.py b/framework/Files.py index 97b7ecf489..a96f288f67 100644 --- a/framework/Files.py +++ b/framework/Files.py @@ -166,14 +166,6 @@ def getLinkedCode(self): """ return self.__linkedModel - def getModelType(self): - """ - Retriever for model type. - @ In, None - @ Out, getLinkedCode, string, string path - """ - return self.modelType - # setting tools # def setPath(self,path): """ @@ -539,7 +531,6 @@ def _readMoreXML(self,node): self.subDirectory = node.attrib.get('subDirectory',"") self.setAbsFile(os.path.join(self.subDirectory,node.text.strip())) self.alias = node.attrib.get('name' ,self.getFilename()) - self.modelType = node.attrib.get('modelType' ,"") def __getstate__(self): """ @@ -551,7 +542,6 @@ def __getstate__(self): stateDict['perturbed' ] = self.perturbed stateDict['subDirectory'] = self.subDirectory stateDict['alias' ] = self.alias - stateDict['modelType' ] = self.modelType return stateDict def __setstate__(self,stateDict): @@ -564,7 +554,6 @@ def __setstate__(self,stateDict): self.perturbed = stateDict['perturbed' ] self.subDirectory = stateDict['subDirectory'] self.alias = stateDict['alias' ] - self.modelType = stateDict['modelType' ] # diff --git a/framework/Steps/IOStep.py b/framework/Steps/IOStep.py index 071f4da340..d233bcf4e6 100644 --- a/framework/Steps/IOStep.py +++ b/framework/Steps/IOStep.py @@ -124,7 +124,7 @@ def _localInitializeStep(self,inDictionary): #XXX switching on the file extension is not very # extendable. Possibly should have a type or something #Past condition: if 'py' == outputs[i].getExt().lower(): - if 'Pyomo' == outputs[i].getModelType(): + if 'Pyomo' == outputs[i].getType(): self.actionType.append('MODEL-PYOMO') else: self.actionType.append('MODEL-FILES') diff --git a/tests/framework/Models/External/serialize_pyomo.xml b/tests/framework/Models/External/serialize_pyomo.xml index 2ddf6a9f2b..ff8c2f568d 100644 --- a/tests/framework/Models/External/serialize_pyomo.xml +++ b/tests/framework/Models/External/serialize_pyomo.xml @@ -38,7 +38,7 @@ - rom_out.py + rom_out.py rom_pickle.pk From e4645396b5e3757a154d33b2a8f6962af921b375 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Tue, 8 Feb 2022 10:38:15 -0700 Subject: [PATCH 31/57] Changed dependencies for pyomo to be optional --- dependencies.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dependencies.xml b/dependencies.xml index a89426503d..885e5a70d1 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -60,11 +60,11 @@ Note all install methods after "main" take 1.1 0.9.39 - - - - - + + + + + remove From 8e3b47a8ab57e9e76a2c8b0b00658af98065232a Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 16 Mar 2022 13:16:53 -0600 Subject: [PATCH 32/57] Restricting pyomo install to version 6.2.0 --- dependencies.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dependencies.xml b/dependencies.xml index 6f11ea3963..c2a9005eac 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -67,8 +67,7 @@ Note all install methods after "main" take 0.9.39 - - + 6.2.0 From 689ed424096544ce9c8dbd4edc4c6f8148de44b0 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 16 Mar 2022 14:01:42 -0600 Subject: [PATCH 33/57] Fixed left conflicts in IOStep.py --- framework/Steps/IOStep.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/framework/Steps/IOStep.py b/framework/Steps/IOStep.py index 6032ac0513..ac8f0980d0 100644 --- a/framework/Steps/IOStep.py +++ b/framework/Steps/IOStep.py @@ -222,7 +222,6 @@ def _localTakeAstepRun(self,inDictionary): # check the ROM is trained first if isinstance(inDictionary['Input'][i],Models.ROM) and not inDictionary['Input'][i].amITrained: self.raiseAnError(RuntimeError,'Pickled rom "%s" was not trained! Train it before pickling and unpickling using a RomTrainer step.' %inDictionary['Input'][i].name) -<<<<<<< HEAD # call the serialize method within the model ## TODO: ADD Deserialization method inDictionary['Input'][i].serialize(outputs[i]) @@ -231,7 +230,6 @@ def _localTakeAstepRun(self,inDictionary): outfile = open(outputs[i].getAbsFile(),"w") outfile.write(inDictionary['Input'][i].writePyomoGreyModel()) outfile.close() -======= fileobj = outputs[i] fileobj.open(mode='wb+') cloudpickle.dump(inDictionary['Input'][i], fileobj, protocol=pickle.HIGHEST_PROTOCOL) @@ -246,7 +244,6 @@ def _localTakeAstepRun(self,inDictionary): fdir = inDictionary['jobHandler'].runInfoDict['FrameworkDir'] fmuexec = FMUexporter(**{'model': inDictionary['Input'][i],'executeMethod': 'evaluate', 'workingDir': outputs[i].getPath(), 'frameworkDir': fdir, 'keepModule': True}) fmuexec.buildFMU(outputs[i].getAbsFile()) ->>>>>>> origin/devel elif self.actionType[i] == 'FILES-MODEL': #inDictionary['Input'][i] is a Files, outputs[i] is ROM or ExternalModel From 073b9208d24e0015ff5082ba113a1ca69e44cb03 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 16 Mar 2022 16:43:30 -0600 Subject: [PATCH 34/57] Incorrectly fixed conflicts in IOStep --- framework/Steps/IOStep.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/framework/Steps/IOStep.py b/framework/Steps/IOStep.py index ac8f0980d0..286309aecd 100644 --- a/framework/Steps/IOStep.py +++ b/framework/Steps/IOStep.py @@ -222,19 +222,17 @@ def _localTakeAstepRun(self,inDictionary): # check the ROM is trained first if isinstance(inDictionary['Input'][i],Models.ROM) and not inDictionary['Input'][i].amITrained: self.raiseAnError(RuntimeError,'Pickled rom "%s" was not trained! Train it before pickling and unpickling using a RomTrainer step.' %inDictionary['Input'][i].name) - # call the serialize method within the model - ## TODO: ADD Deserialization method - inDictionary['Input'][i].serialize(outputs[i]) - - elif self.actionType[i] == 'MODEL-PYOMO': - outfile = open(outputs[i].getAbsFile(),"w") - outfile.write(inDictionary['Input'][i].writePyomoGreyModel()) - outfile.close() fileobj = outputs[i] fileobj.open(mode='wb+') cloudpickle.dump(inDictionary['Input'][i], fileobj, protocol=pickle.HIGHEST_PROTOCOL) fileobj.flush() fileobj.close() + + elif self.actionType[i] == 'MODEL-PYOMO': + outfile = open(outputs[i].getAbsFile(),"w") + outfile.write(inDictionary['Input'][i].writePyomoGreyModel()) + outfile.close() + elif self.actionType[i] == 'MODEL-FMU': #check the ROM is trained first (if ExternalModel no check it is performed) if isinstance(inDictionary['Input'][i],Models.ROM) and not inDictionary['Input'][i].amITrained: From ccf29da78e9cddda40d1e64d69210e62038023cb Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Thu, 17 Mar 2022 09:16:45 -0600 Subject: [PATCH 35/57] Fix test syntax. --- tests/framework/Models/External/tests | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index 078c1604a5..376b03ea76 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -13,6 +13,7 @@ type = 'RavenFramework' input = 'serialize_pyomo.xml' output = 'SerializePyomo/rom_out.py' + [../] [./fmu_ext_model] type = 'RavenFramework' input = 'fmu_ext_model_and_use.xml' From fb67e57b89276196e45adcf57c2551a1a9e3a6fa Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Thu, 17 Mar 2022 09:17:23 -0600 Subject: [PATCH 36/57] Use driver utils setup. --- framework/Models/ROM.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/framework/Models/ROM.py b/framework/Models/ROM.py index 07fcc183d9..8fbefb746f 100644 --- a/framework/Models/ROM.py +++ b/framework/Models/ROM.py @@ -626,7 +626,8 @@ def __init__(self, **kwargs): sys.path.append(self._raven_framework) else: sys.path.append(os.path.join(self._raven_framework,"framework")) - import Driver + from CustomDrivers import DriverUtils as dutils + dutils.doSetup() # de-serialize the ROM self._rom_file = os.path.abspath(self._rom_file) if not os.path.exists(self._rom_file): From 125f4a5f146b61cf04ec49b2584b537e279301ba Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Thu, 17 Mar 2022 09:19:50 -0600 Subject: [PATCH 37/57] Use serialization method. --- framework/Steps/IOStep.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/framework/Steps/IOStep.py b/framework/Steps/IOStep.py index 286309aecd..983f7acf71 100644 --- a/framework/Steps/IOStep.py +++ b/framework/Steps/IOStep.py @@ -222,11 +222,7 @@ def _localTakeAstepRun(self,inDictionary): # check the ROM is trained first if isinstance(inDictionary['Input'][i],Models.ROM) and not inDictionary['Input'][i].amITrained: self.raiseAnError(RuntimeError,'Pickled rom "%s" was not trained! Train it before pickling and unpickling using a RomTrainer step.' %inDictionary['Input'][i].name) - fileobj = outputs[i] - fileobj.open(mode='wb+') - cloudpickle.dump(inDictionary['Input'][i], fileobj, protocol=pickle.HIGHEST_PROTOCOL) - fileobj.flush() - fileobj.close() + inDictionary['Input'][i].serialize(outputs[i]) elif self.actionType[i] == 'MODEL-PYOMO': outfile = open(outputs[i].getAbsFile(),"w") From ee409f2bf4b6768ee440037e2b4302675772c3e0 Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Thu, 17 Mar 2022 16:51:18 -0600 Subject: [PATCH 38/57] Simplifying dependency. --- dependencies.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.xml b/dependencies.xml index c2a9005eac..0dfe20ac12 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -67,7 +67,7 @@ Note all install methods after "main" take 0.9.39 - 6.2.0 + 6.2 From b405b90b1bc8056b40c8cb461ecc188d396c2dc9 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Mon, 21 Mar 2022 09:00:08 -0600 Subject: [PATCH 39/57] Updated establish_conda_env to install pyomo extensions only when pyomo 6.4 is present --- dependencies.xml | 2 +- scripts/establish_conda_env.sh | 2 +- .../gold/SerializePyomo/GrayModelOutput.txt | Bin 2476 -> 0 bytes .../External/gold/SerializePyomo/rom_out.py | 3 ++- .../gold/SerializePyomo/rom_pickle.pk | Bin 8314 -> 7959 bytes tests/framework/Models/External/tests | 7 +++++++ 6 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 tests/framework/Models/External/gold/SerializePyomo/GrayModelOutput.txt diff --git a/dependencies.xml b/dependencies.xml index c2a9005eac..fe64617e63 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -67,7 +67,7 @@ Note all install methods after "main" take 0.9.39 - 6.2.0 + 6.4.0 diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index 7d7e21f7be..191ff248a0 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -109,7 +109,7 @@ function install_libraries() local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset forge)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... conda-forge command: ${COMMAND}; fi ${COMMAND} - if [[ ${COMMAND} == *"pyomo"* ]]; + if [[ ${COMMAND} == *"pyomo-6.4"* ]]; then activate_env #command -v pyomo diff --git a/tests/framework/Models/External/gold/SerializePyomo/GrayModelOutput.txt b/tests/framework/Models/External/gold/SerializePyomo/GrayModelOutput.txt deleted file mode 100644 index d6fab02c3cc39557e9b21e0288154cb3ec64cad6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2476 zcmeHDO;5r=5cS+&F*h_3LWLr5^RC2*#`AVNEhFtrvmcQ9>zxJ@A;z;q*v4$~msc&y> z`|~!P%(CVEYBA5Ud77=VbfJt*Y;vVY36g)H4Rk=ihV%ll!ag`d9DYFu~Zg-VjqmpO|c=KA=16#CFt58+vLmA!;_+ z>+pghItD9+{_fK}q4%cJAs;=|l{gB!1_C-&HUSIp(ORXHv93EX0Swe)O+TnQHKvAE zbdOu`CXg>X;tIkJ)qyoog=`Um)aZ=fJX~ADkR$jOEtRAO`AJFZy_t+O(7TFT_1J^T j_aP@~c9$-*)na-&=~7v*-w8?jWw@ZAH{J1+DM`Kn(xcyC@tmEwMw(d#W^!)ZW<^Nk!ZCTu7Z^2%*+|ij_tPY%wnoy zH3leVqHE4nUqnSih(V2p2T|fBF;O0j3Gu~O6BA>if;5_73Hr~>?v^bmVIOAB{B!32 zzw@2%`}h34;kEVk&()33>3F4nZ_@uEm%07LkM_&8i3j+x+V@*Ce(i`Dzy$@A?d}Ww z86@Whg!j%kwX(UNx?XmWctiPKGDsiFx?TV?Qd8klh_}^!*TVYQKI$hulHX6md5jR2 zTmv#4C>X`i&KAZhu(q*fn#GvbVk||Fo0Q%EV}||YHyxmPW5~^VBin|l8L-R#oK3mH;Jw79+ed~e06he2 z_S2>ec~z)E5y7Ud>53u|B3d%h6h%Vi++2QOR(vQCVjOq=_q2 z1zW0ZibRwV)>KO;rY;lPq)3yHYM@HxRprXgY#1@wkCzeg=QKjdZz>hhhhtWqCWwwX zDY*6lBfHpcS~v(>e0Zx6QRCbZCJHGlI4v-A+IV|&=jdd4QZI|J&$*uxuaKwJj2V=g zR1v5w=mr)9n+Ph9rDMym6$E=&!MzSLM_Uy#<~Nj+1#H6q#052E?JRZE9;4|T_|8r9 zv&|<*BU%pzC0LMQ<<`KFqD+nqH&9S&o0kXsgsv=UXd3R>+aq^Y?K6l7ge4N&AQCpQ zNDM)dB?LPy*aoPsgu4BL^KcdHWSk2?pPpRMmAENBXA^ur4NvoM_P~($2+6}TJ({n= zMP&sG5=K;&6pWCrYl=){72Jgd7~=|?k43QBLHQA#sp_17kAK?oLMXuVu{8gW!|5BH z+_GJ8Q$yl&f2AXlwp#d*>~aH{Z^B{n;Go(xk&) zl?)^eK@XN?0U?TY$<#48MU!>lPcx}L1^m@AaCRvP%`p_?swJ;?v8!V@*WzVVjL{nw{&p= z7&&@V7|t>Hwqn~tspDkWj+cXs6Kcc>`+w2thHDrKdt1@jQhEwoM$zdjJ%vs!vi0G# z&>stmEC|OTNkuju4o~o3mUnkqj27T9P%wr`-g5)k(kN~G5AU3X>EPkdcyGt@yLXp-CDkODn8I{C2c4g+=3Vc=ZHn(NOA@vFx4 L+W&gBCB5ojuRwY; delta 2672 zcmb7`TWl0n7{|M%yR@aHRA@`VZpTJz8>!6YoViH2S%k#KHP&*C&F0L^8O{#v?z%He zQne`rfo(F;HD?S2sKh595TjyvAkm1v_&|7pH{Udx7?o&%Xlx*!*=x6e;LDjg_uu)x z@Bf{1V$(lc!iO5(PjT;sUyHoe^!2ShJ)iA|^`Y%_uKs)j**or2wB%gx@;G+f#3XW4 znHZYHe#~pGw)im$8?N?x^;3e+3em@2#6_9?*!9PKZwX2|J|YL`>hSK+5ttdW%g=cc$3PP(<5QnpqT81Ccb+Goj?^9X$>+!YQGZ}0G*Zthm*}CT zPxV|TmxddM=a>CHQ+H#te~k5$56Ic9j)D*UX6cKeYqYg-TgauOjbCx1D(Jdl@TR6p z0P-^8dG>{pBmo{`EKrG{6*R1&7u)+1cgjO!q=4MA9V0iJ!L>$}Q~)f%GZQKR$QY^6 zGz1NpmV!hbW2J(-nXMcc;AUV{ud%$C&wRM`u`l~dHZB3Mv4f+&4GVK@g?-38mT+x(O9?K?#3Iwtss*)kAswn6pGpB;A3dhSe9FgKujN-zHmHi>Vjty1eEm0E%i8o+{V!TR0w|1;zn%dQIWTxN> za;+S24>rbP-$aIY)mTH(c~KG-9Z6UP*kIb0v8tK^)^rnUX4NYPsvHD1kWI`RdpqBW zRw6!ZOd!XphC36a*GD&YJIlhD6g8}ARqamK9%v_X>`z^CsJnb3`^nOPvU(}#-nfwSQDq7& zLx%z*f))UZszO1tFbjAUDWV|@6;IEWfl4y9*Ruw@H7NL?2qe1pKQv0iYx)-Q{SP~~ z*IZpS0b(lH5|AZ99_T=nG+hOnWC^?^s8}m3y(4L`5p%b!3WA0tjJM<4sA^bYw^hk4#iC`hy?DuQCLq*ff zK9s~h%cj=~*K0`!S6AX-F$a^9h6Npq0+3|@L>{P6)|rqjQ^FWvQ#BT3v@ZgnnKF^r zSYC}K6HnVsaFTP_CE%vGu}l`PKz6?C57@yKW(Qix+gn)~GzAGbZ@*Hi1!*vE$4j*!9p%ZUU|5h5^5j(T z8~8p?P6xk%@$>dYRtB};)vH*wDRj*ab2J_8q?e-| L51Ra2^qGGF?Q^cS diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index 376b03ea76..93174a5283 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -14,6 +14,13 @@ input = 'serialize_pyomo.xml' output = 'SerializePyomo/rom_out.py' [../] + [./pynumero_GBM] + type = 'RavenPython' + input = 'rom_out.py -r rom_pickle.pk -f ../../../.. > GrayModelOutput.txt' + output = 'GrayModelOutput.txt' + python3_only = True + minimum_library_versions = 'pyomo 6.4' + [../] [./fmu_ext_model] type = 'RavenFramework' input = 'fmu_ext_model_and_use.xml' From 8747611e8ae04068fad4d6cda85057fe0e5b837a Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Mon, 21 Mar 2022 09:30:30 -0600 Subject: [PATCH 40/57] Forgot to change a line. --- scripts/establish_conda_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index 191ff248a0..fd940778a4 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -148,7 +148,7 @@ function create_libraries() local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action create --subset forge)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... conda-forge command: ${COMMAND}; fi ${COMMAND} - if [[ ${COMMAND} == *"pyomo"* ]]; + if [[ ${COMMAND} == *"pyomo-6.4"* ]]; then activate_env #command -v pyomo From 3fa762dfc022d1df6cf1e8ce3916009cdd6e1d3b Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Mon, 28 Mar 2022 17:14:30 -0600 Subject: [PATCH 41/57] Changed conditions to match given library installation format. --- scripts/establish_conda_env.sh | 2 +- scripts/library_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index fd940778a4..6c5d3cfdc5 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -148,7 +148,7 @@ function create_libraries() local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action create --subset forge)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... conda-forge command: ${COMMAND}; fi ${COMMAND} - if [[ ${COMMAND} == *"pyomo-6.4"* ]]; + if [[ ${COMMAND} == *"pyomo=6.4"* ]]; then activate_env #command -v pyomo diff --git a/scripts/library_handler.py b/scripts/library_handler.py index 2c43db4405..48dc19d187 100644 --- a/scripts/library_handler.py +++ b/scripts/library_handler.py @@ -431,7 +431,7 @@ def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources, re Reads a single library request node into existing config @ In, libNode, xml.etree.ElementTree.Element, node with library request @ In, config, dict, mapping of existing configuration requests - @ In, toRemove, list, list of library names to be remeoved at the end + @ In, toRemove, list, list of library names to be removed at the end @ In, opSys, str, operating system (not checked) @ In, install, str, installation method (not checked) @ In, addOptional, bool, optional, if True then include optional libraries From 6e75706cfef31491fdfaacde300e03e9d1f82648 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 30 Mar 2022 00:28:15 -0600 Subject: [PATCH 42/57] Created new method to install pynumero. This is a test version. --- dependencies.xml | 12 +++++---- scripts/establish_conda_env.sh | 46 +++++++++++++++++++++------------- scripts/library_handler.py | 12 +++++++-- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/dependencies.xml b/dependencies.xml index 7d5da8838f..515c054762 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -58,7 +58,7 @@ Note all install methods after "main" take - + 1.9 2.9 @@ -67,10 +67,12 @@ Note all install methods after "main" take 0.9.39 - 6.4 - - - + 6.4 + + + + + remove diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index 6c5d3cfdc5..2a8de2f8c0 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -109,15 +109,6 @@ function install_libraries() local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset forge)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... conda-forge command: ${COMMAND}; fi ${COMMAND} - if [[ ${COMMAND} == *"pyomo-6.4"* ]]; - then - activate_env - #command -v pyomo - local PYNUMEROEXT=`echo pyomo download-extensions` - local PYNUMEROBLD=`echo pyomo build-extensions` - ${PYNUMEROEXT} - ${PYNUMEROBLD} || echo "Pyomo build failed" - fi # pip only activate_env if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from PIP-ONLY ...; fi @@ -125,6 +116,20 @@ function install_libraries() if [[ "$PROXY_COMM" != "" ]]; then COMMAND=`echo $COMMAND --proxy $PROXY_COMM`; fi if [[ $ECE_VERBOSE == 0 ]]; then echo ...pip-only command: ${COMMAND}; fi ${COMMAND} + # pyomo only + if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from pyomo ...; fi + local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset pyomo)` + if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi + if [[ ${COMMAND} == *"download-extensions"* ]]; + then + local PYNUMEROEXT=`echo pyomo download-extensions` + ${PYNUMEROEXT} + fi + if [[ ${COMMAND} == *"build-extensions"* ]]; + then + local PYNUMEROBLD=`echo pyomo build-extensions` + ${PYNUMEROBLD} || echo "Pyomo build failed" + fi else # activate the enviroment activate_env @@ -148,15 +153,6 @@ function create_libraries() local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action create --subset forge)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... conda-forge command: ${COMMAND}; fi ${COMMAND} - if [[ ${COMMAND} == *"pyomo=6.4"* ]]; - then - activate_env - #command -v pyomo - local PYNUMEROEXT=`echo pyomo download-extensions` - local PYNUMEROBLD=`echo pyomo build-extensions` - ${PYNUMEROEXT} - ${PYNUMEROBLD} || echo "Pyomo build failed" - fi # pip only activate_env if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from PIP-ONLY ...; fi @@ -164,6 +160,20 @@ function create_libraries() if [[ "$PROXY_COMM" != "" ]]; then COMMAND=`echo $COMMAND --proxy $PROXY_COMM`; fi if [[ $ECE_VERBOSE == 0 ]]; then echo ...pip-only command: ${COMMAND}; fi ${COMMAND} + # pyomo only + if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from pyomo ...; fi + local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset pyomo)` + if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi + if [[ ${COMMAND} == *"download-extensions"* ]]; + then + local PYNUMEROEXT=`echo pyomo download-extensions` + ${PYNUMEROEXT} + fi + if [[ ${COMMAND} == *"build-extensions"* ]]; + then + local PYNUMEROBLD=`echo pyomo build-extensions` + ${PYNUMEROBLD} || echo "Pyomo build failed" + fi else #pip create virtual enviroment local COMMAND=`echo virtualenv $PIP_ENV_LOCATION --python=python` diff --git a/scripts/library_handler.py b/scripts/library_handler.py index 48dc19d187..0efee274e7 100644 --- a/scripts/library_handler.py +++ b/scripts/library_handler.py @@ -378,7 +378,7 @@ def _getInstallMethod(override=None): @ In, override, str, optional, use given method if valid @ Out, install, str, type of install """ - valid = ['conda', 'pip'] #custom? + valid = ['conda', 'pip', 'pyomo'] #custom? if override is not None: if override.lower() not in valid: raise TypeError('Library Handler: Provided override install method not recognized: "{}"! Acceptable options: {}'.format(override, valid)) @@ -529,7 +529,7 @@ def _readDependencies(initFile): help='Chooses whether to (create) a new environment, (install) in existing environment, ' + 'or (list) installation libraries.') condaParser.add_argument('--subset', dest='subset', - choices=('core', 'forge', 'pip'), default='core', + choices=('core', 'forge', 'pip', 'pyomo'), default='core', help='Use subset of installation libraries, divided by source.') pipParser = subParsers.add_parser('pip', help='use pip as installer') @@ -609,6 +609,14 @@ def _readDependencies(initFile): actionArgs = '' addOptional = False limit = ['pip'] + elif args.subset == 'pyomo': + src = '' + installer = 'pyomo' + equals = '==' + equalsTail = '.*' + actionArgs = '' + addOptional = args.addOptional + limit = ['pyomo'] libs = getRequiredLibs(useOS=args.useOS, installMethod='conda', addOptional=addOptional, From a611af1fb8901c2c3cfb7de3ab857eda5b7667ba Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 30 Mar 2022 00:51:34 -0600 Subject: [PATCH 43/57] Confirmed that HERON is blocking from pyomo 6.4 being installed. HERON must catch up with pyomo 6.4 before this PR can be merged. --- dependencies.xml | 14 +++++++------- tests/framework/Models/External/tests | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dependencies.xml b/dependencies.xml index 515c054762..6fb371307a 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -58,7 +58,7 @@ Note all install methods after "main" take - + 1.9 2.9 @@ -67,12 +67,12 @@ Note all install methods after "main" take 0.9.39 - 6.4 - - - - - + 6.4 + + + + + remove diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index 93174a5283..c3cb212c45 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -20,6 +20,7 @@ output = 'GrayModelOutput.txt' python3_only = True minimum_library_versions = 'pyomo 6.4' + required_libraries = 'cmake glpk ipopt cyipopt' [../] [./fmu_ext_model] type = 'RavenFramework' From 8394aa2c4ee422e267233807d6bebb33438f200b Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 30 Mar 2022 09:44:30 -0600 Subject: [PATCH 44/57] Changed install script for pynumero according to comments provided. Required libraries for pynumero test were changed. --- dependencies.xml | 7 +++---- scripts/establish_conda_env.sh | 22 ++++++---------------- tests/framework/Models/External/tests | 2 +- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/dependencies.xml b/dependencies.xml index 6fb371307a..25fd908ad2 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -67,12 +67,11 @@ Note all install methods after "main" take 0.9.39 - 6.4 + 6.4 - + - - + remove diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index 2a8de2f8c0..2928bd85a3 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -120,15 +120,10 @@ function install_libraries() if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from pyomo ...; fi local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset pyomo)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi - if [[ ${COMMAND} == *"download-extensions"* ]]; + if [[ ${COMMAND} == *"pyomo-extensions"* ]]; then - local PYNUMEROEXT=`echo pyomo download-extensions` - ${PYNUMEROEXT} - fi - if [[ ${COMMAND} == *"build-extensions"* ]]; - then - local PYNUMEROBLD=`echo pyomo build-extensions` - ${PYNUMEROBLD} || echo "Pyomo build failed" + pyomo download-extensions + pyomo build-extensions || echo "Pyomo build failed" fi else # activate the enviroment @@ -164,15 +159,10 @@ function create_libraries() if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from pyomo ...; fi local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset pyomo)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi - if [[ ${COMMAND} == *"download-extensions"* ]]; - then - local PYNUMEROEXT=`echo pyomo download-extensions` - ${PYNUMEROEXT} - fi - if [[ ${COMMAND} == *"build-extensions"* ]]; + if [[ ${COMMAND} == *"pyomo-extensions"* ]]; then - local PYNUMEROBLD=`echo pyomo build-extensions` - ${PYNUMEROBLD} || echo "Pyomo build failed" + pyomo download-extensions + pyomo build-extensions || echo "Pyomo build failed" fi else #pip create virtual enviroment diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index c3cb212c45..8092a16490 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -20,7 +20,7 @@ output = 'GrayModelOutput.txt' python3_only = True minimum_library_versions = 'pyomo 6.4' - required_libraries = 'cmake glpk ipopt cyipopt' + required_libraries = 'ipopt' [../] [./fmu_ext_model] type = 'RavenFramework' From 4166f3f8bc0053ba1e261aa0f6172865cfd27a28 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 30 Mar 2022 11:23:14 -0600 Subject: [PATCH 45/57] Modified user manual according to recent changes. --- doc/user_manual/rom.tex | 8 ++++++++ framework/Models/Model.py | 2 +- framework/Steps/IOStep.py | 3 --- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index 22d8d5a20e..ee67dd14ad 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -209,6 +209,14 @@ \subsubsection{SerializePyomo} When loading ROMs from file, the user is expected to know the I/O of a ROM and loading of the ROM file must use \xmlNode{IOStep}. +To verify the output files in the given example below are runnable with GrayBoxModel, several optional RAVEN dependencies are required. +The following are the required optional libraries: pyomo 6.4 or over, cmake, glpk, ipopt, cyipopt, and pyomo-extensions. + +If the user does not have a setup pyomo and pynumero environment, it is recommended to activate RAVEN libraries using the following command +on the terminal: + +source scripts/establish_conda_env.sh --load + \textbf{Example:} For this example the external model is trained and further loaded as `out\_rom'. The loaded trained model is separately processed to `rom\_out.py' and `rom\_pickle.pk'. \begin{lstlisting}[style=XML,morekeywords={name,subType}] diff --git a/framework/Models/Model.py b/framework/Models/Model.py index 3b35ec6960..fc1606fff3 100644 --- a/framework/Models/Model.py +++ b/framework/Models/Model.py @@ -387,7 +387,7 @@ def initialize(self,runInfo,inputs,initDict=None): def serialize(self,fileObjIn,**kwargs): """ This method is the base class method that is aimed to serialize the model (and derived) instances. - @ In, fileo, str or File object, the filename of the output serialized binary file or the RAVEN File instance + @ In, fileObjIn, str or File object, the filename of the output serialized binary file or the RAVEN File instance @ In, kwargs, dict, dictionary of options that the derived class might require @ Out, None """ diff --git a/framework/Steps/IOStep.py b/framework/Steps/IOStep.py index 983f7acf71..0baa81d8fd 100644 --- a/framework/Steps/IOStep.py +++ b/framework/Steps/IOStep.py @@ -121,9 +121,6 @@ def _localInitializeStep(self,inDictionary): elif isinstance(inDictionary['Input'][i], (Models.ROM, Models.ExternalModel)): # ... file if isinstance(outputs[i],Files.File): - #XXX switching on the file extension is not very - # extendable. Possibly should have a type or something - #Past condition: if 'py' == outputs[i].getExt().lower(): if 'Pyomo' == outputs[i].getType(): self.actionType.append('MODEL-PYOMO') if 'FMU' == outputs[i].getType().upper(): From 5dc2869039dbcbe1960e510f5d50cae1633c18a5 Mon Sep 17 00:00:00 2001 From: "Joshua J. Cogliati" Date: Wed, 30 Mar 2022 12:55:15 -0600 Subject: [PATCH 46/57] Putting shell command in lstlisting. --- doc/user_manual/rom.tex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index ee67dd14ad..1d12f0aab2 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -215,7 +215,9 @@ \subsubsection{SerializePyomo} If the user does not have a setup pyomo and pynumero environment, it is recommended to activate RAVEN libraries using the following command on the terminal: +\begin{lstlisting} source scripts/establish_conda_env.sh --load +\end{lstlisting} \textbf{Example:} For this example the external model is trained and further loaded as `out\_rom'. The loaded trained model is separately processed to `rom\_out.py' and `rom\_pickle.pk'. From 6a4a66b2bdfa91f0ae71de4dab1beb96fc18f9f9 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL <78757932+yoshiurr-INL@users.noreply.github.com> Date: Mon, 18 Apr 2022 13:08:45 -0600 Subject: [PATCH 47/57] Update rom.tex Changed indent. --- doc/user_manual/rom.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index ce8cfeed9e..b5b972325f 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -221,7 +221,7 @@ \subsubsection{SerializePyomo} on the terminal: \begin{lstlisting} -source scripts/establish_conda_env.sh --load + source scripts/establish_conda_env.sh --load \end{lstlisting} \textbf{Example:} For this example the external model is trained and further loaded as `out\_rom'. The loaded trained model is separately From 58d09e5177f21aa804b4688359ad92c55f762aa4 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 20 Jul 2022 10:44:15 -0600 Subject: [PATCH 48/57] Added exceptions to pyomo command download-extension --- scripts/establish_conda_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index 2928bd85a3..65a34a3cb4 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -122,7 +122,7 @@ function install_libraries() if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi if [[ ${COMMAND} == *"pyomo-extensions"* ]]; then - pyomo download-extensions + pyomo download-extensions || echo "Pyomo download failed" pyomo build-extensions || echo "Pyomo build failed" fi else @@ -161,7 +161,7 @@ function create_libraries() if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi if [[ ${COMMAND} == *"pyomo-extensions"* ]]; then - pyomo download-extensions + pyomo download-extensions || echo "Pyomo download failed" pyomo build-extensions || echo "Pyomo build failed" fi else From 6a755f898da1acfeff9c79e4471fd4f30334f25b Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 20 Jul 2022 11:18:44 -0600 Subject: [PATCH 49/57] Solved conflict in IOStep.py --- ravenframework/Steps/IOStep.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ravenframework/Steps/IOStep.py b/ravenframework/Steps/IOStep.py index 52dab68bf9..037018d2ef 100644 --- a/ravenframework/Steps/IOStep.py +++ b/ravenframework/Steps/IOStep.py @@ -218,8 +218,12 @@ def _localTakeAstepRun(self,inDictionary): ## pickle the ROM # check the ROM is trained first if isinstance(inDictionary['Input'][i],Models.ROM) and not inDictionary['Input'][i].amITrained: - self.raiseAnError(RuntimeError,'Pickled rom "%s" was not trained! Train it before pickling and unpickling using a RomTrainer step.' %inDictionary['Input'][i].name) - inDictionary['Input'][i].serialize(outputs[i]) + self.raiseAnError(RuntimeError, f'Pickled rom "{inDictionary["Input"][i].name}" was not trained! Train it before pickling and unpickling using a RomTrainer step.') + fileobj = outputs[i] + fileobj.open(mode='wb+') + cloudpickle.dump(inDictionary['Input'][i], fileobj, protocol=pickle.HIGHEST_PROTOCOL) + fileobj.flush() + fileobj.close() elif self.actionType[i] == 'MODEL-PYOMO': outfile = open(outputs[i].getAbsFile(),"w") From 5ac466a80d8540e6ce3139502613e3bd8224c603 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 20 Jul 2022 11:22:46 -0600 Subject: [PATCH 50/57] Solved conflict in IOStep.py the second time. --- ravenframework/Steps/IOStep.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ravenframework/Steps/IOStep.py b/ravenframework/Steps/IOStep.py index 037018d2ef..b67e0bbefc 100644 --- a/ravenframework/Steps/IOStep.py +++ b/ravenframework/Steps/IOStep.py @@ -214,8 +214,8 @@ def _localTakeAstepRun(self,inDictionary): romModel.writePointwiseData(outputs[i]) elif self.actionType[i] == 'MODEL-FILES': - #inDictionary['Input'][i] is a ROM, outputs[i] is Files - ## pickle the ROM + # inDictionary['Input'][i] is a ROM, outputs[i] is Files + # pickle the ROM # check the ROM is trained first if isinstance(inDictionary['Input'][i],Models.ROM) and not inDictionary['Input'][i].amITrained: self.raiseAnError(RuntimeError, f'Pickled rom "{inDictionary["Input"][i].name}" was not trained! Train it before pickling and unpickling using a RomTrainer step.') From 3ea5c10417f8224fc3316f16569e71d556da12f8 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 20 Jul 2022 15:14:00 -0600 Subject: [PATCH 51/57] Rebased, and updated code based on commit conflicts and suggestions provided. User manual updates will come later. --- dependencies.xml | 12 ++--- ravenframework/Models/ROM.py | 14 +++-- .../Models/External/gold/GrayModelOutput.txt | 20 ------- .../External/gold/GreyModelOutput_cyipopt.txt | 51 ++++++++++++++++++ .../External/gold/SerializePyomo/rom_out.py | 12 +++-- .../gold/SerializePyomo/rom_pickle.pk | Bin 8073 -> 8145 bytes tests/framework/Models/External/rom_out.py | 12 +++-- tests/framework/Models/External/rom_pickle.pk | Bin 8073 -> 8145 bytes tests/framework/Models/External/tests | 3 +- 9 files changed, 83 insertions(+), 41 deletions(-) delete mode 100644 tests/framework/Models/External/gold/GrayModelOutput.txt create mode 100644 tests/framework/Models/External/gold/GreyModelOutput_cyipopt.txt diff --git a/dependencies.xml b/dependencies.xml index de2feb842d..46f740e888 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -58,7 +58,7 @@ Note all install methods after "main" take - + 1.9 2.9 @@ -67,11 +67,11 @@ Note all install methods after "main" take 0.9.39 - 6.4 - - - - + 6.4 + + + + diff --git a/ravenframework/Models/ROM.py b/ravenframework/Models/ROM.py index e610c8a967..de72e18d47 100644 --- a/ravenframework/Models/ROM.py +++ b/ravenframework/Models/ROM.py @@ -601,7 +601,7 @@ def writeXML(self, what='all'): def writePyomoGreyModel(self): """ - Called by the OutStreamPrint object to cause the ROM to print itself + @ In, None, called by the OutStreamPrint object to cause the ROM to print itself @ Out, xml, xmlUtils.StaticXmlElement, written meta """ template = r""" @@ -612,6 +612,7 @@ def writePyomoGreyModel(self): from pyomo.contrib.pynumero.asl import AmplInterface from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP import os, sys, pickle +from contextlib import redirect_stdout # RAVEN ROM PYOMO GREY MODEL CLASS class ravenROM(ExternalGreyBoxModel): @@ -709,10 +710,13 @@ def pyomoModel(ex_model): print(results) count = 0 lines = str(results).split("\n") - textfile = open("GrayModelOutput.txt", "w") - for element in lines: - textfile.write(element + "\n") - textfile.close() + with open ("GreyModelOutput_cyipopt.txt", "w") as textfile: + with redirect_stdout(textfile): + print("---------------------< Output Summary >---------------------" + "\n") + for element in lines: + textfile.write(element + "\n") + print("---------------------< Optimization Results >---------------------" + "\n") + concreteModel.pprint() """ return template diff --git a/tests/framework/Models/External/gold/GrayModelOutput.txt b/tests/framework/Models/External/gold/GrayModelOutput.txt deleted file mode 100644 index d1bd6a0eb6..0000000000 --- a/tests/framework/Models/External/gold/GrayModelOutput.txt +++ /dev/null @@ -1,20 +0,0 @@ - -Problem: -- Name: unknown - Lower bound: -inf - Upper bound: 0.3587964554159516 - Number of objectives: 1 - Number of constraints: 1 - Number of variables: 3 - Number of binary variables: 0 - Number of integer variables: 0 - Number of continuous variables: 3 - Sense: minimize -Solver: -- Name: cyipopt - Status: ok - Return code: 0 - Message: b'Algorithm terminated successfully at a locally optimal point, satisfying the convergence tolerances (can be specified by options).' - Wallclock time: 0.15934334900000002 - Termination condition: optimal - diff --git a/tests/framework/Models/External/gold/GreyModelOutput_cyipopt.txt b/tests/framework/Models/External/gold/GreyModelOutput_cyipopt.txt new file mode 100644 index 0000000000..2401df74c4 --- /dev/null +++ b/tests/framework/Models/External/gold/GreyModelOutput_cyipopt.txt @@ -0,0 +1,51 @@ +---------------------< Output Summary >--------------------- + + +Problem: +- Name: unknown + Lower bound: -inf + Upper bound: 0.3587964554159516 + Number of objectives: 1 + Number of constraints: 1 + Number of variables: 3 + Number of binary variables: 0 + Number of integer variables: 0 + Number of continuous variables: 3 + Sense: minimize +Solver: +- Name: cyipopt + Status: ok + Return code: 0 + Message: b'Algorithm terminated successfully at a locally optimal point, satisfying the convergence tolerances (can be specified by options).' + Wallclock time: 0.14550156800000003 + Termination condition: optimal + +---------------------< Optimization Results >--------------------- + +1 Objective Declarations + obj : Size=1, Index=None, Active=True + Key : Active : Sense : Expression + None : True : minimize : egb.outputs[ans] + +1 ExternalGreyBoxBlock Declarations + egb : Size=1, Index=None, Active=True + 2 Set Declarations + _input_names_set : Size=1, Index=None, Ordered=Insertion + Key : Dimen : Domain : Size : Members + None : 1 : Any : 2 : {'y1', 'y2'} + _output_names_set : Size=1, Index=None, Ordered=Insertion + Key : Dimen : Domain : Size : Members + None : 1 : Any : 1 : {'ans',} + + 2 Var Declarations + inputs : Size=2, Index=egb._input_names_set + Key : Lower : Value : Upper : Fixed : Stale : Domain + y1 : 0.0 : 0.938770350603 : 1.0 : False : False : Reals + y2 : 0.0 : 0.938770350603 : 1.0 : False : False : Reals + outputs : Size=1, Index=egb._output_names_set + Key : Lower : Value : Upper : Fixed : Stale : Domain + ans : 0.358796465406 : 0.358796455416 : 0.975309912028 : False : False : Reals + + 4 Declarations: _input_names_set inputs _output_names_set outputs + +2 Declarations: egb obj diff --git a/tests/framework/Models/External/gold/SerializePyomo/rom_out.py b/tests/framework/Models/External/gold/SerializePyomo/rom_out.py index 8dca587320..ada7c169c9 100644 --- a/tests/framework/Models/External/gold/SerializePyomo/rom_out.py +++ b/tests/framework/Models/External/gold/SerializePyomo/rom_out.py @@ -6,6 +6,7 @@ from pyomo.contrib.pynumero.asl import AmplInterface from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP import os, sys, pickle +from contextlib import redirect_stdout # RAVEN ROM PYOMO GREY MODEL CLASS class ravenROM(ExternalGreyBoxModel): @@ -103,7 +104,10 @@ def pyomoModel(ex_model): print(results) count = 0 lines = str(results).split("\n") - textfile = open("GrayModelOutput.txt", "w") - for element in lines: - textfile.write(element + "\n") - textfile.close() + with open ("GreyModelOutput_cyipopt.txt", "w") as textfile: + with redirect_stdout(textfile): + print("---------------------< Output Summary >---------------------" + "\n") + for element in lines: + textfile.write(element + "\n") + print("---------------------< Optimization Results >---------------------" + "\n") + concreteModel.pprint() diff --git a/tests/framework/Models/External/gold/SerializePyomo/rom_pickle.pk b/tests/framework/Models/External/gold/SerializePyomo/rom_pickle.pk index 6fb7f4c9af6100caa92dd70f67ad1373334b5df9..5868202c165b6bee79f025d9d5372dbd36508151 100644 GIT binary patch delta 2323 zcmb7`U1%It6vwkM-K3^T)0XDD=~8S>tEqQp?tBRnu^?i!q-h9MY~#Im?)1)>?k>Bt zfhchmX-&$|MrK-2d?;GA4`L~Sg2XR;D1uL7#aAB$K~N#oP_ee2nf=IaYOTKPoH=Lb z+}}O-od3-H@HfLvuQZ*x@v&52jY@%*Hdo(xACgNw=gn#H^G;(f5cO+J&j7z{O!w}0p- z^NdW9To{C6DLCD{tMemqX{6zJC>CE2$HKvp#7HCDECpXT@7(sKSlZXJ@6Xm>iVeRG zhbJH|&Wra7lth=Ht>wwY)x|?C_assZk}TQ85@Sros*MezN*Yx;mo!3TCIv68e>m~m z;^p-N>k=!$ckOoPkAvbTzipc5Xtbx9gIUf(aU(deu}hXEQ@2!!D7LL)Sy!oHTDH!> zVNglpdbr<`p8$}4_MM~nI<`h&?yf2RssC(p`lbdld^c8<(zEMb291F4EIt)Lt!m7E7g7|f-_AAQQnd-0xwa1Hmp z<0N;Kdv(;rFp)w`3sVU=HJM06Rx~PU3Py}Dtz7foWp}g=J6XyF(2IX<*_F5}pZf$n zzJ_OW)|t+Dqa+90^mwk0i%GhrAlu+XS1>0G2F27h3CXrbvBBhOFC2-ndMo8@ug?j5 z!6#c^E5=7Fu2%Z_&ShMC*2#SnbzBOOsmv5rvLp#xoD)T{EF@d9qF@~vt6WpHuoqaV;D9U34!wyk@Pw+xle{nRE_eOIW#`F11Znk#H zDm@DeuWgRCyZR@IuncUd8j(!KFmy3Rrm9HTwh*J3suhvivh2QGyED*;mBjUR9}cJ&%+QpsZp4O$`t=D4e!Rp~xV?2DN0u3>7hETb8L+fR^2p6`ryhxCk7v z3b;~aMWf&MFS8$by*q08{zn}**^USACum;Av-N|8DA%zj*)k&-BU_=St;rnPm?|nE zj8_}@%?N)(MGwRo=!1d2w|B>VFn9Ncy%GG-*ELa6)^sL!40P|slCp}j_#-SMQ<6cA zba)FO6-M0VvP3bFKvo&4l3a*Lsvu|KV4v>qNt8-6e^(Os96Vl~@IqOf2F1kbaL|q3 zD{Z}oD-NrQME_Fcq&5u_vChoSIyWA7&HZVD zv7W|&#jIN01z!wCG?*BTA&n%B#%RKWZwBI{zL=O86OwPz~Gut&@elE{RsW3jt zOXXa#5O^77KvM@$P^IF`u>W1=K7T&*Wa?9YboprK$6lt)NS>4eKM-^N`Q_WYKM|h| zwHyn?xmSacpnoVevk7sWwu#1(f*8y*q&n0;)BZr}^6Zc8TUt}s{2QI__)q=fXKy7i z(_%SSIr_u|XMvdaIvy!hk33AueDvr92gYN;RN!T8jVWAVhNHL!)*WQqmO~7eAk)?i z=D5LdC1krI+fywqQ$fGD>VLCpI(_qmxaL3BWp#^NvISPEf`^M(+z>aXVd-?&u5`HM zU+mgbi-6imK1a%Rm|yC#iKUr_iy*vWVhb_Lu_$6HRcwrGoh-!8He!oEgdxVnUqj!D zzagI2Hc3i3Lh~HXvJlaJ8CDRZmmc`_n zqL~B)cCf=Of)tnAmdXs>Q8d*`WZBg~lPh2(46lC9NMqRnQ{Kuj$Y42~sn2%8?6h13 z*TVpPP#ogbDX4(sBTA&5+!k&W%UYG)@FaWv$JcDF-DtcwEF%1l{hU;)CEl#6O)O#{ z1L>H!Sk)NS3A34{>eNzfUB?OfL$Grw#???&udT6JAg+0HWhk1Qo8#qdMeuBa9D$L4 zWz9!hW9efo*kQ*=AaS&lG~g!Y{kgT94@G=;kCM`}ywa3&N{FixL~P5z zhNduw*#=MOhMVXz^{s$yR#~?_Ra*)}?uXA8$+c!8H(9J4ASF0Wg_oMKG=)-xFjrlM zO^XxFb}iM^G>s9>&{2Z*RD`v$p;~A2p1|k-zWx=cHgTy@rFWKUaXvHq#+slsZga*R z+aw%egecWiY+y_^!Yl=u3tGv4Yb6E+G=e|7-`JGcq?P4KxxC zG@WA%&N2IXkCEza97Nt5zmW8qA9^25ARDJx!rH_{2r{d>n3^CaCZ@xo>tF||+(wut zsh)1o#MYJrFcEoL?g>Y+k9%NzTO<3k8@DEK|3ksmZmu$(D?o`dKGEzR%q^rV4u>8| zF{R8#h7MvnjzM*95nr!k@Y4U7(p>x-93twug(weAKM`V3CPd7K)`K z(9xc+w@eYYa#ynzlTu0$G#zZKmWmNL$u@PyIH7Th*TL_|*_f;jb_xvr_NLwxkeyhP zOrC~s6P{|^cKXHGn?b(}Ye$AZ6f<%!T*Ilb!Oh46@l)7PX5`cHQ|RLvF&MrUx@YZ- zI2n#Z-Dbq8aCp|=yJh5HxW|r{L+i*dVK^1Yui$n#@LDHHzRCm0ytnMA8o)&hc$La9 r%---------------------" + "\n") + for element in lines: + textfile.write(element + "\n") + print("---------------------< Optimization Results >---------------------" + "\n") + concreteModel.pprint() diff --git a/tests/framework/Models/External/rom_pickle.pk b/tests/framework/Models/External/rom_pickle.pk index 6fb7f4c9af6100caa92dd70f67ad1373334b5df9..5868202c165b6bee79f025d9d5372dbd36508151 100644 GIT binary patch delta 2323 zcmb7`U1%It6vwkM-K3^T)0XDD=~8S>tEqQp?tBRnu^?i!q-h9MY~#Im?)1)>?k>Bt zfhchmX-&$|MrK-2d?;GA4`L~Sg2XR;D1uL7#aAB$K~N#oP_ee2nf=IaYOTKPoH=Lb z+}}O-od3-H@HfLvuQZ*x@v&52jY@%*Hdo(xACgNw=gn#H^G;(f5cO+J&j7z{O!w}0p- z^NdW9To{C6DLCD{tMemqX{6zJC>CE2$HKvp#7HCDECpXT@7(sKSlZXJ@6Xm>iVeRG zhbJH|&Wra7lth=Ht>wwY)x|?C_assZk}TQ85@Sros*MezN*Yx;mo!3TCIv68e>m~m z;^p-N>k=!$ckOoPkAvbTzipc5Xtbx9gIUf(aU(deu}hXEQ@2!!D7LL)Sy!oHTDH!> zVNglpdbr<`p8$}4_MM~nI<`h&?yf2RssC(p`lbdld^c8<(zEMb291F4EIt)Lt!m7E7g7|f-_AAQQnd-0xwa1Hmp z<0N;Kdv(;rFp)w`3sVU=HJM06Rx~PU3Py}Dtz7foWp}g=J6XyF(2IX<*_F5}pZf$n zzJ_OW)|t+Dqa+90^mwk0i%GhrAlu+XS1>0G2F27h3CXrbvBBhOFC2-ndMo8@ug?j5 z!6#c^E5=7Fu2%Z_&ShMC*2#SnbzBOOsmv5rvLp#xoD)T{EF@d9qF@~vt6WpHuoqaV;D9U34!wyk@Pw+xle{nRE_eOIW#`F11Znk#H zDm@DeuWgRCyZR@IuncUd8j(!KFmy3Rrm9HTwh*J3suhvivh2QGyED*;mBjUR9}cJ&%+QpsZp4O$`t=D4e!Rp~xV?2DN0u3>7hETb8L+fR^2p6`ryhxCk7v z3b;~aMWf&MFS8$by*q08{zn}**^USACum;Av-N|8DA%zj*)k&-BU_=St;rnPm?|nE zj8_}@%?N)(MGwRo=!1d2w|B>VFn9Ncy%GG-*ELa6)^sL!40P|slCp}j_#-SMQ<6cA zba)FO6-M0VvP3bFKvo&4l3a*Lsvu|KV4v>qNt8-6e^(Os96Vl~@IqOf2F1kbaL|q3 zD{Z}oD-NrQME_Fcq&5u_vChoSIyWA7&HZVD zv7W|&#jIN01z!wCG?*BTA&n%B#%RKWZwBI{zL=O86OwPz~Gut&@elE{RsW3jt zOXXa#5O^77KvM@$P^IF`u>W1=K7T&*Wa?9YboprK$6lt)NS>4eKM-^N`Q_WYKM|h| zwHyn?xmSacpnoVevk7sWwu#1(f*8y*q&n0;)BZr}^6Zc8TUt}s{2QI__)q=fXKy7i z(_%SSIr_u|XMvdaIvy!hk33AueDvr92gYN;RN!T8jVWAVhNHL!)*WQqmO~7eAk)?i z=D5LdC1krI+fywqQ$fGD>VLCpI(_qmxaL3BWp#^NvISPEf`^M(+z>aXVd-?&u5`HM zU+mgbi-6imK1a%Rm|yC#iKUr_iy*vWVhb_Lu_$6HRcwrGoh-!8He!oEgdxVnUqj!D zzagI2Hc3i3Lh~HXvJlaJ8CDRZmmc`_n zqL~B)cCf=Of)tnAmdXs>Q8d*`WZBg~lPh2(46lC9NMqRnQ{Kuj$Y42~sn2%8?6h13 z*TVpPP#ogbDX4(sBTA&5+!k&W%UYG)@FaWv$JcDF-DtcwEF%1l{hU;)CEl#6O)O#{ z1L>H!Sk)NS3A34{>eNzfUB?OfL$Grw#???&udT6JAg+0HWhk1Qo8#qdMeuBa9D$L4 zWz9!hW9efo*kQ*=AaS&lG~g!Y{kgT94@G=;kCM`}ywa3&N{FixL~P5z zhNduw*#=MOhMVXz^{s$yR#~?_Ra*)}?uXA8$+c!8H(9J4ASF0Wg_oMKG=)-xFjrlM zO^XxFb}iM^G>s9>&{2Z*RD`v$p;~A2p1|k-zWx=cHgTy@rFWKUaXvHq#+slsZga*R z+aw%egecWiY+y_^!Yl=u3tGv4Yb6E+G=e|7-`JGcq?P4KxxC zG@WA%&N2IXkCEza97Nt5zmW8qA9^25ARDJx!rH_{2r{d>n3^CaCZ@xo>tF||+(wut zsh)1o#MYJrFcEoL?g>Y+k9%NzTO<3k8@DEK|3ksmZmu$(D?o`dKGEzR%q^rV4u>8| zF{R8#h7MvnjzM*95nr!k@Y4U7(p>x-93twug(weAKM`V3CPd7K)`K z(9xc+w@eYYa#ynzlTu0$G#zZKmWmNL$u@PyIH7Th*TL_|*_f;jb_xvr_NLwxkeyhP zOrC~s6P{|^cKXHGn?b(}Ye$AZ6f<%!T*Ilb!Oh46@l)7PX5`cHQ|RLvF&MrUx@YZ- zI2n#Z-Dbq8aCp|=yJh5HxW|r{L+i*dVK^1Yui$n#@LDHHzRCm0ytnMA8o)&hc$La9 r% Date: Thu, 21 Jul 2022 09:32:34 -0600 Subject: [PATCH 52/57] Change dependencies to check for cyipopt and added it to required libraries for pynumero test case. --- dependencies.xml | 12 ++++++------ tests/framework/Models/External/tests | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dependencies.xml b/dependencies.xml index 46f740e888..e2f97e6009 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -58,7 +58,7 @@ Note all install methods after "main" take - + 1.9 2.9 @@ -67,11 +67,11 @@ Note all install methods after "main" take 0.9.39 - 6.4 - - - - + 6.4 + + + + diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index 0255a4c442..862b77c04c 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -20,6 +20,7 @@ output = 'GreyModelOutput_cyipopt.txt' python3_only = True minimum_library_versions = 'pyomo 6.4' + required_libraries = 'cyipopt' [../] [./fmu_ext_model] type = 'RavenFramework' From 78a4bea463dfc76d3ee0593d47f3013cdae4288d Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Mon, 15 Aug 2022 13:46:56 -0600 Subject: [PATCH 53/57] Updated user manual. --- doc/user_manual/rom.tex | 61 +++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index b5b972325f..c19faece9f 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -194,43 +194,70 @@ \subsection{ROM} \subsubsection{SerializePyomo} \label{subsubsec:serializepyomo} -For the purpose of exporting RAVEN trained ROMs in Pyomo format, the platform to print the Pyomo python file is established. -The capabilities implemented enable to retrieve sampled and trained ROMs, pickle (serialize), and output python files as -defined in the \xmlNode{IOStep}. +For the purpose of exporting RAVEN trained ROMs in an universal-accessible format for Pyomo to solve, the platform to prepare +the python syntax including the process to setup concrete models and constraints is created. The python syntax allows to +to retrieve pickled (serialized) ROMs and solve the defined concrete model via Pynumero's GreyBoxModel, an extension of Pyomo. -The trained file is copied and processed separately to output two files required to run -Pyomo. The python output file contains the infrastructure inheriting the External Grey Box Model and others included in -the Pyomo contributions package Pynumero to print a file with Pyomo syntax. The `.pk' output file is the pickled version of the -trained ROM and can be ran with the python output file. +Two files are required to run the optimization: the pickled rom and the generated python file. The workflow to create both files is +provided in the example below located at the end of this subsection. The following command line is the defined format to interface both files: -Since the code developed in ROMs is not a subtype, the code activation is triggered by file type. To ensure developed Pyomo code -generator is utilized in ROMs, one of the file extensions in the output of the \xmlNode{IOStep} must be `.py'. The subtype of the -ROM model can be of any kind and is the user's decision to choose the training method. +\begin{lstlisting} + python3 printed_python_file.py -r rom_pickled_file.pk -f location_of_raven_framework +\end{lstlisting} + +The file names `printed_python_file' and `rom_pickled_file' are arbitrary names, and are not requirements. +The `location_of_raven_framework' is the relative path to where `raven_framework' is located. The created python file includes modules from +RAVEN and is required dealing with derivatives for GreyBoxModel to solve. The rest of the command line syntax `-r' and `-f' are place holders for +the python file to locate where the pickled ROMs and the python file itself are. +When running the command a text file named `GreyModelOutput_cyipopt.txt' will be created showing the results of the optimization. + +Since the code developed in ROMs is not a subtype, the code activation for python interface file print is triggered by adding the input subnode with type `Pyomo': + +\begin{lstlisting}[style=XML,morekeywords={name,subType}] + rom_out.py +\end{lstlisting} + +To verify the output files in the given test file in 'raven/tests/framework/Models/External/serialize_pyomo.xml' and the associated example optimization case +`python3 rom_out.py -r rom_pickle.pk -f ../../../..' are runnable with GreyBoxModel, several optional RAVEN dependencies are required. +The following are the required optional libraries: `pyomo 6.4' or over, `cmake', `glpk', `ipopt', `cyipopt', and `pyomo-extensions'. Although almost all required +libraries are standard packages (available on conda or pipy install), the `pyomo-extensions' is a custom library created by RAVEN developers to install `Pynumero'. +It functions to run the following commands if `pyomo-extensions' is part of the RAVEN library installation: -When loading ROMs from file, the user is expected to know the I/O of a ROM and loading of the ROM file must use \xmlNode{IOStep}. +\begin{lstlisting} + pyomo download-extensions + pyomo build-extensions +\end{lstlisting} -To verify the output files in the given example below are runnable with GrayBoxModel, several optional RAVEN dependencies are required. -The following are the required optional libraries: pyomo 6.4 or over, cmake, glpk, ipopt, cyipopt, and pyomo-extensions. The syntax to run -trained ROMs and GayBoxModel is the following: +For this reason, libraries `Pyomo' and `cmake' are required for successful `Pynumero' installation. Note, although all RAVEN dependency installs will +be placed in the RAVEN library directory, the current `Pyomo' version saves extensions on the local python local directory. +Other libraries are optimization solvers for `Pyomo' and `Pynumero'. Make sure the RAVEN conda library is activated when testing the example cases. +The method to include optional RAVEN libraries is by adding the `optional' flag to the RAVEN library installation command: \begin{lstlisting} - python3 python_output_file.py -r rom_pickle_file.pk -f location_of_raven_framework + cd raven + ./scripts/establish_conda_env.sh --install --optional='pyomo cmake ipopt cyipopt pyomo-extensions' \end{lstlisting} -If the user does not have a setup pyomo and pynumero environment, it is recommended to activate RAVEN libraries using the following command +If the user plans to run the optimization case outside of RAVEN libraries and the pickled ROM has already been generated, instead install `pyomo', `cmake', +`glpk', `ipopt', and `cyipopt' to your system. After this is done, on a terminal, run the `Pyomo' download and extension command introduced earlier. +This will install `Pynumero' to your system. + +If the user plans to use the RAVEN library, it is recommended to activate RAVEN libraries using the following command on the terminal: \begin{lstlisting} source scripts/establish_conda_env.sh --load \end{lstlisting} +Make sure that the optional libraries are installed before running the pickled ROM and printed python file. + \textbf{Example:} For this example the external model is trained and further loaded as `out\_rom'. The loaded trained model is separately processed to `rom\_out.py' and `rom\_pickle.pk'. \begin{lstlisting}[style=XML,morekeywords={name,subType}] ... - rom_out.py + rom_out.py rom_pickle.pk ... From 17a607af0c17f25d64783667a821590404ba01c8 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Mon, 15 Aug 2022 13:56:41 -0600 Subject: [PATCH 54/57] Corrected previous commit for user manual. --- doc/user_manual/rom.tex | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index c19faece9f..46c26aaf46 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -217,9 +217,20 @@ \subsubsection{SerializePyomo} rom_out.py \end{lstlisting} -To verify the output files in the given test file in 'raven/tests/framework/Models/External/serialize_pyomo.xml' and the associated example optimization case -`python3 rom_out.py -r rom_pickle.pk -f ../../../..' are runnable with GreyBoxModel, several optional RAVEN dependencies are required. -The following are the required optional libraries: `pyomo 6.4' or over, `cmake', `glpk', `ipopt', `cyipopt', and `pyomo-extensions'. Although almost all required +To verify the output files in the given test file in: + +\begin{lstlisting}[style=XML,morekeywords={name,subType}] + raven/tests/framework/Models/External/serialize_pyomo.xml +\end{lstlisting} + +and the associated example optimization case: + +\begin{lstlisting}[style=XML,morekeywords={name,subType}] + python3 rom_out.py -r rom_pickle.pk -f ../../../.. +\end{lstlisting} + +are runnable with GreyBoxModel, several optional RAVEN dependencies are required. The following are the required +optional libraries: `pyomo 6.4' or over, `cmake', `glpk', `ipopt', `cyipopt', and `pyomo-extensions'. Although almost all required libraries are standard packages (available on conda or pipy install), the `pyomo-extensions' is a custom library created by RAVEN developers to install `Pynumero'. It functions to run the following commands if `pyomo-extensions' is part of the RAVEN library installation: From f97f245e7cf74c5eaf14f70f5994f6d123dc9b63 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Mon, 15 Aug 2022 16:02:46 -0600 Subject: [PATCH 55/57] Corrected back slash error. --- doc/user_manual/rom.tex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index 46c26aaf46..b7d60d8b60 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -205,11 +205,11 @@ \subsubsection{SerializePyomo} python3 printed_python_file.py -r rom_pickled_file.pk -f location_of_raven_framework \end{lstlisting} -The file names `printed_python_file' and `rom_pickled_file' are arbitrary names, and are not requirements. -The `location_of_raven_framework' is the relative path to where `raven_framework' is located. The created python file includes modules from +The file names `printed\_python\_file' and `rom\_pickled_file' are arbitrary names, and are not requirements. +The `location\_of\_raven\_framework' is the relative path to where `raven\_framework' is located. The created python file includes modules from RAVEN and is required dealing with derivatives for GreyBoxModel to solve. The rest of the command line syntax `-r' and `-f' are place holders for the python file to locate where the pickled ROMs and the python file itself are. -When running the command a text file named `GreyModelOutput_cyipopt.txt' will be created showing the results of the optimization. +When running the command a text file named `GreyModelOutput\_cyipopt.txt' will be created showing the results of the optimization. Since the code developed in ROMs is not a subtype, the code activation for python interface file print is triggered by adding the input subnode with type `Pyomo': From 10df9ae9a124c95cab7cb2a11bc88024cb692152 Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Tue, 16 Aug 2022 07:50:22 -0600 Subject: [PATCH 56/57] Added clause concerning C++ libraries. --- doc/user_manual/rom.tex | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index b7d60d8b60..0b501a483f 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -205,7 +205,7 @@ \subsubsection{SerializePyomo} python3 printed_python_file.py -r rom_pickled_file.pk -f location_of_raven_framework \end{lstlisting} -The file names `printed\_python\_file' and `rom\_pickled_file' are arbitrary names, and are not requirements. +The file names `printed\_python\_file' and `rom\_pickled\_file' are arbitrary names, and are not requirements. The `location\_of\_raven\_framework' is the relative path to where `raven\_framework' is located. The created python file includes modules from RAVEN and is required dealing with derivatives for GreyBoxModel to solve. The rest of the command line syntax `-r' and `-f' are place holders for the python file to locate where the pickled ROMs and the python file itself are. @@ -260,7 +260,14 @@ \subsubsection{SerializePyomo} source scripts/establish_conda_env.sh --load \end{lstlisting} -Make sure that the optional libraries are installed before running the pickled ROM and printed python file. +Make sure that the optional libraries are installed before running the pickled ROM and printed python file. Regardless of whether `Pynumero' is +installed inside or outside RAVEN libraries, depending on the OS, C++ libraries may be outdated for `Pynumero' to run. It has been reported that +the following OS has failed to run `Pynumero': + +\begin{itemize} + \item CentOS 7, 8 + \item Ubuntu 16, 18 +\end{itemize} \textbf{Example:} For this example the external model is trained and further loaded as `out\_rom'. The loaded trained model is separately processed to `rom\_out.py' and `rom\_pickle.pk'. From 1852eb91825dbdaca8c390f82b9670771997f55e Mon Sep 17 00:00:00 2001 From: yoshiurr-INL Date: Wed, 17 Aug 2022 16:30:20 -0600 Subject: [PATCH 57/57] Edited files based on previous and current provided comments --- doc/user_manual/rom.tex | 15 ++++++++++----- ravenframework/Models/Model.py | 2 +- ravenframework/Models/ROM.py | 9 +++++++-- ravenframework/Steps/IOStep.py | 2 +- scripts/establish_conda_env.sh | 2 +- .../External/gold/SerializePyomo/rom_out.py | 9 +++++++-- .../framework/Models/External/serialize_pyomo.xml | 2 +- 7 files changed, 28 insertions(+), 13 deletions(-) diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index 0b501a483f..71bc0635c6 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -197,19 +197,24 @@ \subsubsection{SerializePyomo} For the purpose of exporting RAVEN trained ROMs in an universal-accessible format for Pyomo to solve, the platform to prepare the python syntax including the process to setup concrete models and constraints is created. The python syntax allows to to retrieve pickled (serialized) ROMs and solve the defined concrete model via Pynumero's GreyBoxModel, an extension of Pyomo. +Details on how GreyBoxModel works can be found in the following site: + +\begin{lstlisting} + https://pyomo.readthedocs.io/en/stable/contributed_packages/pynumero/index.html +\end{lstlisting} Two files are required to run the optimization: the pickled rom and the generated python file. The workflow to create both files is provided in the example below located at the end of this subsection. The following command line is the defined format to interface both files: \begin{lstlisting} - python3 printed_python_file.py -r rom_pickled_file.pk -f location_of_raven_framework + python3 printed_python_file.py -r rom_pickled_file.pk -f location_of_raven_framework -order order_of_derivative \end{lstlisting} -The file names `printed\_python\_file' and `rom\_pickled\_file' are arbitrary names, and are not requirements. +The file names `printed\_python\_file', `rom\_pickled\_file', and `order\_of\_derivative' are arbitrary names, and are not requirements. The `location\_of\_raven\_framework' is the relative path to where `raven\_framework' is located. The created python file includes modules from -RAVEN and is required dealing with derivatives for GreyBoxModel to solve. The rest of the command line syntax `-r' and `-f' are place holders for -the python file to locate where the pickled ROMs and the python file itself are. -When running the command a text file named `GreyModelOutput\_cyipopt.txt' will be created showing the results of the optimization. +RAVEN and is required dealing with derivatives for GreyBoxModel to solve. The rest of the command line syntax `-r', `-f', `-order' are place holders for +the python file to locate where the pickled ROMs and the python file itself are. Caution, make sure the `order\_of\_derivative' is an integer. If not provided, +the default value is 1. When running the command a text file named `GreyModelOutput\_cyipopt.txt' will be created showing the results of the optimization. Since the code developed in ROMs is not a subtype, the code activation for python interface file print is triggered by adding the input subnode with type `Pyomo': diff --git a/ravenframework/Models/Model.py b/ravenframework/Models/Model.py index 7f3ae44687..bd2defe771 100644 --- a/ravenframework/Models/Model.py +++ b/ravenframework/Models/Model.py @@ -395,7 +395,7 @@ def serialize(self,fileObjIn,**kwargs): if isinstance(fileObjIn,str): fileObj = open(filename, mode='wb+') else: - fileObj = fileObjIn + fileObj = fileObjIn # if issues occur add 'isintance(fileObjIn,Files)'. fileObj.open(mode='wb+') cloudpickle.dump(self,fileObj, protocol=pickle.HIGHEST_PROTOCOL) fileObj.flush() diff --git a/ravenframework/Models/ROM.py b/ravenframework/Models/ROM.py index de72e18d47..372955c0ef 100644 --- a/ravenframework/Models/ROM.py +++ b/ravenframework/Models/ROM.py @@ -620,6 +620,7 @@ class ravenROM(ExternalGreyBoxModel): def __init__(self, **kwargs): self._rom_file = kwargs.get("rom_file") self._raven_framework = kwargs.get("raven_framework") + self._order = kwargs.get("order") self._raven_framework = os.path.abspath(self._raven_framework) if not os.path.exists(self._raven_framework): raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') @@ -671,7 +672,7 @@ def evaluate_outputs(self): def evaluate_jacobian_outputs(self): request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} - derivatives = self.rom.derivatives(request) + derivatives = self.rom.derivatives(request, order=self._order) jac = np.zeros((self._n_outputs, self._n_inputs)) for tc, target in enumerate(self._output_names): for fc, feature in enumerate(self._input_names): @@ -700,7 +701,11 @@ def pyomoModel(ex_model): rom_file = sys.argv[cnt+1] if item.lower() == "-f": raven_framework = sys.argv[cnt+1] - ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework}) + if item.lower() == "-order": + order = int(sys.argv[cnt+1]) + else: + order = 1 + ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework,'order':order}) concreteModel = pyomoModel(ext_model) ### here you should implement the optimization problem ### diff --git a/ravenframework/Steps/IOStep.py b/ravenframework/Steps/IOStep.py index 831ca7f9c3..6e928234f4 100644 --- a/ravenframework/Steps/IOStep.py +++ b/ravenframework/Steps/IOStep.py @@ -111,7 +111,7 @@ def _localInitializeStep(self,inDictionary): elif isinstance(inDictionary['Input'][i], (Models.ROM, Models.ExternalModel)): # ... file if isinstance(outputs[i],Files.File): - if 'Pyomo' == outputs[i].getType(): + if 'PYOMO' == outputs[i].getType().upper(): self.actionType.append('MODEL-PYOMO') if 'FMU' == outputs[i].getType().upper(): self.actionType.append('MODEL-FMU') diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index 65a34a3cb4..765b5fbf16 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -120,7 +120,7 @@ function install_libraries() if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from pyomo ...; fi local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset pyomo)` if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi - if [[ ${COMMAND} == *"pyomo-extensions"* ]]; + if [[ ${COMMAND} == *"pyomo-extensions"* ]]; # If pip package is created for pynumero, delete this command and add to pip dependencies then pyomo download-extensions || echo "Pyomo download failed" pyomo build-extensions || echo "Pyomo build failed" diff --git a/tests/framework/Models/External/gold/SerializePyomo/rom_out.py b/tests/framework/Models/External/gold/SerializePyomo/rom_out.py index ada7c169c9..f7ffbf57e8 100644 --- a/tests/framework/Models/External/gold/SerializePyomo/rom_out.py +++ b/tests/framework/Models/External/gold/SerializePyomo/rom_out.py @@ -14,6 +14,7 @@ class ravenROM(ExternalGreyBoxModel): def __init__(self, **kwargs): self._rom_file = kwargs.get("rom_file") self._raven_framework = kwargs.get("raven_framework") + self._order = kwargs.get("order") self._raven_framework = os.path.abspath(self._raven_framework) if not os.path.exists(self._raven_framework): raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') @@ -65,7 +66,7 @@ def evaluate_outputs(self): def evaluate_jacobian_outputs(self): request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} - derivatives = self.rom.derivatives(request) + derivatives = self.rom.derivatives(request, order=self._order) jac = np.zeros((self._n_outputs, self._n_inputs)) for tc, target in enumerate(self._output_names): for fc, feature in enumerate(self._input_names): @@ -94,7 +95,11 @@ def pyomoModel(ex_model): rom_file = sys.argv[cnt+1] if item.lower() == "-f": raven_framework = sys.argv[cnt+1] - ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework}) + if item.lower() == "-order": + order = int(sys.argv[cnt+1]) + else: + order = 1 + ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework,'order':order}) concreteModel = pyomoModel(ext_model) ### here you should implement the optimization problem ### diff --git a/tests/framework/Models/External/serialize_pyomo.xml b/tests/framework/Models/External/serialize_pyomo.xml index ff8c2f568d..a293999b10 100644 --- a/tests/framework/Models/External/serialize_pyomo.xml +++ b/tests/framework/Models/External/serialize_pyomo.xml @@ -38,7 +38,7 @@ - rom_out.py + rom_out.py rom_pickle.pk