diff --git a/dependencies.xml b/dependencies.xml index e893a7a12c..e2f97e6009 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -64,8 +64,14 @@ Note all install methods after "main" take 1.1 + 0.9.39 + 6.4 + + + + diff --git a/doc/user_manual/rom.tex b/doc/user_manual/rom.tex index f17eb72874..71bc0635c6 100644 --- a/doc/user_manual/rom.tex +++ b/doc/user_manual/rom.tex @@ -191,3 +191,121 @@ \subsection{ROM} %%%%% ROM Model - TensorFlow-Keras Interface %%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \input{kerasROM.tex} + +\subsubsection{SerializePyomo} +\label{subsubsec:serializepyomo} +For the purpose of exporting RAVEN trained ROMs in an universal-accessible format for Pyomo to solve, the platform to prepare +the python syntax including the process to setup concrete models and constraints is created. The python syntax allows to +to retrieve pickled (serialized) ROMs and solve the defined concrete model via Pynumero's GreyBoxModel, an extension of Pyomo. +Details on how GreyBoxModel works can be found in the following site: + +\begin{lstlisting} + https://pyomo.readthedocs.io/en/stable/contributed_packages/pynumero/index.html +\end{lstlisting} + +Two files are required to run the optimization: the pickled rom and the generated python file. The workflow to create both files is +provided in the example below located at the end of this subsection. The following command line is the defined format to interface both files: + +\begin{lstlisting} + python3 printed_python_file.py -r rom_pickled_file.pk -f location_of_raven_framework -order order_of_derivative +\end{lstlisting} + +The file names `printed\_python\_file', `rom\_pickled\_file', and `order\_of\_derivative' are arbitrary names, and are not requirements. +The `location\_of\_raven\_framework' is the relative path to where `raven\_framework' is located. The created python file includes modules from +RAVEN and is required dealing with derivatives for GreyBoxModel to solve. The rest of the command line syntax `-r', `-f', `-order' are place holders for +the python file to locate where the pickled ROMs and the python file itself are. Caution, make sure the `order\_of\_derivative' is an integer. If not provided, +the default value is 1. When running the command a text file named `GreyModelOutput\_cyipopt.txt' will be created showing the results of the optimization. + +Since the code developed in ROMs is not a subtype, the code activation for python interface file print is triggered by adding the input subnode with type `Pyomo': + +\begin{lstlisting}[style=XML,morekeywords={name,subType}] + rom_out.py +\end{lstlisting} + +To verify the output files in the given test file in: + +\begin{lstlisting}[style=XML,morekeywords={name,subType}] + raven/tests/framework/Models/External/serialize_pyomo.xml +\end{lstlisting} + +and the associated example optimization case: + +\begin{lstlisting}[style=XML,morekeywords={name,subType}] + python3 rom_out.py -r rom_pickle.pk -f ../../../.. +\end{lstlisting} + +are runnable with GreyBoxModel, several optional RAVEN dependencies are required. The following are the required +optional libraries: `pyomo 6.4' or over, `cmake', `glpk', `ipopt', `cyipopt', and `pyomo-extensions'. Although almost all required +libraries are standard packages (available on conda or pipy install), the `pyomo-extensions' is a custom library created by RAVEN developers to install `Pynumero'. +It functions to run the following commands if `pyomo-extensions' is part of the RAVEN library installation: + +\begin{lstlisting} + pyomo download-extensions + pyomo build-extensions +\end{lstlisting} + +For this reason, libraries `Pyomo' and `cmake' are required for successful `Pynumero' installation. Note, although all RAVEN dependency installs will +be placed in the RAVEN library directory, the current `Pyomo' version saves extensions on the local python local directory. +Other libraries are optimization solvers for `Pyomo' and `Pynumero'. Make sure the RAVEN conda library is activated when testing the example cases. +The method to include optional RAVEN libraries is by adding the `optional' flag to the RAVEN library installation command: + +\begin{lstlisting} + cd raven + ./scripts/establish_conda_env.sh --install --optional='pyomo cmake ipopt cyipopt pyomo-extensions' +\end{lstlisting} + +If the user plans to run the optimization case outside of RAVEN libraries and the pickled ROM has already been generated, instead install `pyomo', `cmake', +`glpk', `ipopt', and `cyipopt' to your system. After this is done, on a terminal, run the `Pyomo' download and extension command introduced earlier. +This will install `Pynumero' to your system. + +If the user plans to use the RAVEN library, it is recommended to activate RAVEN libraries using the following command +on the terminal: + +\begin{lstlisting} + source scripts/establish_conda_env.sh --load +\end{lstlisting} + +Make sure that the optional libraries are installed before running the pickled ROM and printed python file. Regardless of whether `Pynumero' is +installed inside or outside RAVEN libraries, depending on the OS, C++ libraries may be outdated for `Pynumero' to run. It has been reported that +the following OS has failed to run `Pynumero': + +\begin{itemize} + \item CentOS 7, 8 + \item Ubuntu 16, 18 +\end{itemize} + +\textbf{Example:} For this example the external model is trained and further loaded as `out\_rom'. The loaded trained model is separately +processed to `rom\_out.py' and `rom\_pickle.pk'. +\begin{lstlisting}[style=XML,morekeywords={name,subType}] + + ... + + rom_out.py + rom_pickle.pk + + ... + + ... + + ... + attenuate + ... + samples + + + samples + out_rom + + + out_rom + out_rom.py + + + out_rom + out_pickle.pk + + ... + + ... + +\end{lstlisting} diff --git a/ravenframework/BaseClasses/BaseEntity.py b/ravenframework/BaseClasses/BaseEntity.py index b39b1a30b7..877d8ad820 100644 --- a/ravenframework/BaseClasses/BaseEntity.py +++ b/ravenframework/BaseClasses/BaseEntity.py @@ -246,7 +246,6 @@ def _validateSolutionExportVariables(self, solutionExport): # get acceptable names fromSolnExport = set(self.getSolutionExportVariableNames()) acceptable = set(self._formatSolutionExportVariableNames(fromSolnExport)) - # remove registered solution export names first remaining = requested - acceptable # anything remaining is unknown! diff --git a/ravenframework/Models/Model.py b/ravenframework/Models/Model.py index b1f35ddf83..bd2defe771 100644 --- a/ravenframework/Models/Model.py +++ b/ravenframework/Models/Model.py @@ -13,6 +13,9 @@ # limitations under the License. """ Module where the base class and the specialization of different type of Model are + +@author crisrab, alfoa + """ #External Modules------------------------------------------------------------------------------------ import copy @@ -20,6 +23,7 @@ import abc import sys import importlib +import pickle #External Modules End-------------------------------------------------------------------------------- #Internal Modules------------------------------------------------------------------------------------ @@ -43,7 +47,6 @@ def loadFromPlugins(cls): """ cls.plugins = importlib.import_module(".ModelPlugInFactory","ravenframework.Models") - @classmethod def getInputSpecification(cls): """ @@ -381,6 +384,23 @@ def initialize(self,runInfo,inputs,initDict=None): """ pass + def serialize(self,fileObjIn,**kwargs): + """ + This method is the base class method that is aimed to serialize the model (and derived) instances. + @ In, fileObjIn, str or File object, the filename of the output serialized binary file or the RAVEN File instance + @ In, kwargs, dict, dictionary of options that the derived class might require + @ Out, None + """ + import cloudpickle + if isinstance(fileObjIn,str): + fileObj = open(filename, mode='wb+') + else: + fileObj = fileObjIn # if issues occur add 'isintance(fileObjIn,Files)'. + fileObj.open(mode='wb+') + cloudpickle.dump(self,fileObj, protocol=pickle.HIGHEST_PROTOCOL) + fileObj.flush() + fileObj.close() + @abc.abstractmethod def createNewInput(self,myInput,samplerType,**kwargs): """ diff --git a/ravenframework/Models/ROM.py b/ravenframework/Models/ROM.py index 3bdfd0ef1b..372955c0ef 100644 --- a/ravenframework/Models/ROM.py +++ b/ravenframework/Models/ROM.py @@ -108,7 +108,6 @@ def __init__(self): """ super().__init__() self.amITrained = False # boolean flag, is the ROM trained? - self.supervisedEngine = None # dict of ROM instances (== number of targets => keys are the targets) self.printTag = 'ROM MODEL' # label self.cvInstanceName = None # the name of Cross Validation instance self.cvInstance = None # Instance of provided cross validation @@ -417,6 +416,38 @@ def evaluate(self, request): resultsDict[k] = np.atleast_1d(v) return resultsDict + def derivatives(self, request, feats = None, order=1): + """ + This method is aimed to evaluate the derivatives using this ROM + The differentiation method depends on the specific ROM. Numerical + differentiation is available by default for any ROM. Some ROMs (e.g. + neural networks) use automatic differentiation. + @ In, request, dict, coordinate of the central point + @ In, feats, list, optional, list of features we need to compute the + derivative for (default all) + @ In, order, int, order of the derivative (1 = first, 2 = second, etc.). Max 10 + @ Out, derivatives, dict, the dict containing the derivatives for each target + ({'d(feature1)/d(target1)':np.array(size 1 or n_ts), + 'd(feature1)/d(target1)':np.array(...)}. For example, + {"x1/y1":np.array(...),"x2/y1":np.array(...),etc.} + """ + if self.pickled: + self.raiseAnError(RuntimeError,'ROM "', self.name, '" has not been loaded yet! Use an IOStep to load it.') + if not self.amITrained: + self.raiseAnError(RuntimeError, "ROM ", self.name, " has not been trained yet and, consequentially, can not be evaluated!") + derivatives = {} + if self.segment: + derivatives = mathUtils.derivatives(self.supervisedContainer[0].evaluate, request, var=feats, n=order) + else: + for rom in self.supervisedContainer: + sliceEvaluation = mathUtils.derivatives(rom.evaluate, request, var=feats, n=order) + if len(list(derivatives.keys())) == 0: + derivatives.update(sliceEvaluation) + else: + for key in derivatives.keys(): + derivatives[key] = np.append(derivatives[key],sliceEvaluation[key]) + return derivatives + def _externalRun(self,inRun): """ Method that performs the actual run of the imported external model (separated from run method for parallelization purposes) @@ -567,3 +598,133 @@ def writeXML(self, what='all'): engines[0].writeXMLPreamble(xml) engines[0].writeXML(xml) return xml + + def writePyomoGreyModel(self): + """ + @ In, None, called by the OutStreamPrint object to cause the ROM to print itself + @ Out, xml, xmlUtils.StaticXmlElement, written meta + """ + template = r""" +# MODEL GENERATED BY RAVEN (raven.inl.gov) +import pyomo.environ as pyo +from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock +from pyomo.contrib.pynumero.dependencies import (numpy as np) +from pyomo.contrib.pynumero.asl import AmplInterface +from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP +import os, sys, pickle +from contextlib import redirect_stdout +# RAVEN ROM PYOMO GREY MODEL CLASS + +class ravenROM(ExternalGreyBoxModel): + + def __init__(self, **kwargs): + self._rom_file = kwargs.get("rom_file") + self._raven_framework = kwargs.get("raven_framework") + self._order = kwargs.get("order") + self._raven_framework = os.path.abspath(self._raven_framework) + if not os.path.exists(self._raven_framework): + raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') + if os.path.isdir(os.path.join(self._raven_framework,"ravenframework")): + sys.path.append(self._raven_framework) + else: + raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') + from ravenframework.CustomDrivers import DriverUtils as dutils + dutils.doSetup() + # de-serialize the ROM + self._rom_file = os.path.abspath(self._rom_file) + if not os.path.exists(self._rom_file): + raise IOError('The serialized (binary) file has not been found in location "' + str(self._rom_file)+'" !') + self.rom = pickle.load(open(self._rom_file, mode='rb')) + # + self.settings = self.rom.getInitParams() + # get input names + self._input_names = self.settings.get('Features') + # n_inputs + self._n_inputs = len(self._input_names) + # get output names + self._output_names = self.settings.get('Target') + # n_outputs + self._n_outputs = len(self._output_names) + # input values storage + self._input_values = np.zeros(self._n_inputs, dtype=np.float64) + + def return_train_values(self, feat): + return self.rom.trainingSet.get(feat) + + def input_names(self): + return self._input_names + + def output_names(self): + return self._output_names + + def set_input_values(self, input_values): + assert len(input_values) == self._n_inputs + np.copyto(self._input_values, input_values) + + def evaluate_equality_constraints(self): + raise NotImplementedError('This method should not be called for this model.') + + def evaluate_outputs(self): + request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + outs = self.rom.evaluate(request) + eval_outputs = np.asarray([outs[k].flatten() for k in self._output_names], dtype=np.float64) + return eval_outputs.flatten() + + def evaluate_jacobian_outputs(self): + request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + derivatives = self.rom.derivatives(request, order=self._order) + jac = np.zeros((self._n_outputs, self._n_inputs)) + for tc, target in enumerate(self._output_names): + for fc, feature in enumerate(self._input_names): + jac[tc,fc] = derivatives['d{}|d{}'.format(target, feature)] + return jac + +def pyomoModel(ex_model): + m = pyo.ConcreteModel() + m.egb = ExternalGreyBoxBlock() + m.egb.set_external_model(ex_model) + for inp in ex_model.input_names(): + m.egb.inputs[inp].value = np.mean(ex_model.return_train_values(inp)) + m.egb.inputs[inp].setlb(np.min(ex_model.return_train_values(inp))) + m.egb.inputs[inp].setub(np.max(ex_model.return_train_values(inp))) + for out in ex_model.output_names(): + m.egb.outputs[out].value = np.mean(ex_model.return_train_values(out)) + m.egb.outputs[out].setlb(np.min(ex_model.return_train_values(out))) + m.egb.outputs[out].setub(np.max(ex_model.return_train_values(out))) + m.obj = pyo.Objective(expr=m.egb.outputs[out]) + + return m + +if __name__ == '__main__': + for cnt, item in enumerate(sys.argv): + if item.lower() == "-r": + rom_file = sys.argv[cnt+1] + if item.lower() == "-f": + raven_framework = sys.argv[cnt+1] + if item.lower() == "-order": + order = int(sys.argv[cnt+1]) + else: + order = 1 + ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework,'order':order}) + concreteModel = pyomoModel(ext_model) + ### here you should implement the optimization problem + ### + solver = pyo.SolverFactory('cyipopt') + solver.config.options['hessian_approximation'] = 'limited-memory' + results = solver.solve(concreteModel) + print(results) + count = 0 + lines = str(results).split("\n") + with open ("GreyModelOutput_cyipopt.txt", "w") as textfile: + with redirect_stdout(textfile): + print("---------------------< Output Summary >---------------------" + "\n") + for element in lines: + textfile.write(element + "\n") + print("---------------------< Optimization Results >---------------------" + "\n") + concreteModel.pprint() +""" + + return template + + + diff --git a/ravenframework/Steps/IOStep.py b/ravenframework/Steps/IOStep.py index 726bc60027..6e928234f4 100644 --- a/ravenframework/Steps/IOStep.py +++ b/ravenframework/Steps/IOStep.py @@ -111,6 +111,8 @@ def _localInitializeStep(self,inDictionary): elif isinstance(inDictionary['Input'][i], (Models.ROM, Models.ExternalModel)): # ... file if isinstance(outputs[i],Files.File): + if 'PYOMO' == outputs[i].getType().upper(): + self.actionType.append('MODEL-PYOMO') if 'FMU' == outputs[i].getType().upper(): self.actionType.append('MODEL-FMU') else: @@ -206,6 +208,12 @@ def _localTakeAstepRun(self, inDictionary): cloudpickle.dump(inDictionary['Input'][i], fileobj, protocol=pickle.HIGHEST_PROTOCOL) fileobj.flush() fileobj.close() + + elif self.actionType[i] == 'MODEL-PYOMO': + outfile = open(outputs[i].getAbsFile(),"w") + outfile.write(inDictionary['Input'][i].writePyomoGreyModel()) + outfile.close() + elif self.actionType[i] == 'MODEL-FMU': #check the ROM is trained first (if ExternalModel no check it is performed) if isinstance(inDictionary['Input'][i],Models.ROM) and not inDictionary['Input'][i].amITrained: diff --git a/ravenframework/utils/mathUtils.py b/ravenframework/utils/mathUtils.py index 7824f93fbd..1cb2ef997c 100644 --- a/ravenframework/utils/mathUtils.py +++ b/ravenframework/utils/mathUtils.py @@ -976,6 +976,79 @@ def angleBetweenVectors(a, b): return ang +def partialDerivative(f, x0, var, n = 1, h = None, target = None): + """ + Compute the n-th partial derivative of function f + with respect variable var (numerical differentation). + The derivative is computed with a central difference + approximation. + @ In, f, instance, the function to differentiate (format f(d) where d is a dictionary) + @ In, x0, dict, the dictionary containing the x0 coordinate + @ In, var, str, the variable of the resulting partial derivative + @ In, n, int, optional, the order of the derivative. (max 10) + @ In, h, float, optional, the step size, default automatically computed + @ In, target, str, optional, the target output in case the "f" returns a dict of outputs. Default (takes the first) + @ Out, deriv, float, the partial derivative of function f + """ + import numdifftools as nd + assert(n <= 10) + def func(x, var, target=None): + """ + Simple function wrapper for using scipy + @ In, x, float, the point at which the nth derivative is found + @ In, var, str, the variable in the dictionary x0 corresponding + to the part derivative to compute + @ In, target, str, optional, the target output in case the "f" returns a dict of outputs. Default (takes the first) + @ Out, func, float, the evaluated function + """ + d = copy.copy(x0) + d[var] = x + out = f(d) + if isinstance(out, dict): + return list(out.values())[0] if target is None else out[target] + else: + return out + + do = nd.Derivative(func, order=n) + deriv = do(x0[var],var,target) + return deriv + +def derivatives(f, x0, var = None, n = 1, h = None, target=None): + """ + Compute the n-th partial derivative of function f + with respect variable var (numerical differentation). + The derivative is computed with a central difference + approximation. + @ In, f, instance, the function to differentiate (format f(d) where d is a dictionary) + @ In, x0, dict, the dictionary containing the x0 coordinate {key:scalar or array of len(1)} + @ In, var, list, optional, the list of variables of the resulting partial derivative (if None, compute all) + @ In, n, int, optional, the order of the derivative. (max 10) + @ In, h, float, optional, the step size, default automatically computed + @ In, target, str, optional, the target output in case the "f" returns a dict of outputs. Default (all targets) + @ Out, deriv, dict, the partial derivative of function f + """ + assert(n <= 10) + assert(isinstance(var, list) or isinstance(var, type(None))) + #XXX Possibly this should only be done sometimes + for key in x0: + x0[key] = np.atleast_1d(x0[key]) + assert(len(list(x0.values())[0]) == 1) + #assert(first(x0.values()).size == 1) + targets = [target] + if target is None: + checkWorking = f(x0) + if isinstance(checkWorking, dict): + targets = checkWorking.keys() + deriv = {} + for t in targets: + for variable in (x0.keys() if var is None else var): + name = variable if t is None else "d{}|d{}".format(t, variable) + deriv[name] = partialDerivative(f, x0, variable, n = n, h = h, target=t) + return deriv + + + + # utility function for defaultdict def giveZero(): """ diff --git a/scripts/establish_conda_env.sh b/scripts/establish_conda_env.sh index b68169ad9c..765b5fbf16 100755 --- a/scripts/establish_conda_env.sh +++ b/scripts/establish_conda_env.sh @@ -116,6 +116,15 @@ function install_libraries() if [[ "$PROXY_COMM" != "" ]]; then COMMAND=`echo $COMMAND --proxy $PROXY_COMM`; fi if [[ $ECE_VERBOSE == 0 ]]; then echo ...pip-only command: ${COMMAND}; fi ${COMMAND} + # pyomo only + if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from pyomo ...; fi + local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset pyomo)` + if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi + if [[ ${COMMAND} == *"pyomo-extensions"* ]]; # If pip package is created for pynumero, delete this command and add to pip dependencies + then + pyomo download-extensions || echo "Pyomo download failed" + pyomo build-extensions || echo "Pyomo build failed" + fi else # activate the enviroment activate_env @@ -146,6 +155,15 @@ function create_libraries() if [[ "$PROXY_COMM" != "" ]]; then COMMAND=`echo $COMMAND --proxy $PROXY_COMM`; fi if [[ $ECE_VERBOSE == 0 ]]; then echo ...pip-only command: ${COMMAND}; fi ${COMMAND} + # pyomo only + if [[ $ECE_VERBOSE == 0 ]]; then echo ... Installing libraries from pyomo ...; fi + local COMMAND=`echo $($PYTHON_COMMAND ${RAVEN_LIB_HANDLER} ${INSTALL_OPTIONAL} ${OSOPTION} conda --action install --subset pyomo)` + if [[ $ECE_VERBOSE == 0 ]]; then echo ... pyomo command: ${COMMAND}; fi + if [[ ${COMMAND} == *"pyomo-extensions"* ]]; + then + pyomo download-extensions || echo "Pyomo download failed" + pyomo build-extensions || echo "Pyomo build failed" + fi else #pip create virtual enviroment local COMMAND=`echo virtualenv $PIP_ENV_LOCATION --python=python` diff --git a/scripts/library_handler.py b/scripts/library_handler.py index efce6988fc..a894abef8b 100644 --- a/scripts/library_handler.py +++ b/scripts/library_handler.py @@ -378,7 +378,7 @@ def _getInstallMethod(override=None): @ In, override, str, optional, use given method if valid @ Out, install, str, type of install """ - valid = ['conda', 'pip'] #custom? + valid = ['conda', 'pip', 'pyomo'] #custom? if override is not None: if override.lower() not in valid: raise TypeError('Library Handler: Provided override install method not recognized: "{}"! Acceptable options: {}'.format(override, valid)) @@ -431,7 +431,7 @@ def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources, re Reads a single library request node into existing config @ In, libNode, xml.etree.ElementTree.Element, node with library request @ In, config, dict, mapping of existing configuration requests - @ In, toRemove, list, list of library names to be remeoved at the end + @ In, toRemove, list, list of library names to be removed at the end @ In, opSys, str, operating system (not checked) @ In, install, str, installation method (not checked) @ In, addOptional, bool, optional, if True then include optional libraries @@ -529,7 +529,7 @@ def _readDependencies(initFile): help='Chooses whether to (create) a new environment, (install) in existing environment, ' + 'or (list) installation libraries.') condaParser.add_argument('--subset', dest='subset', - choices=('core', 'forge', 'pip'), default='core', + choices=('core', 'forge', 'pip', 'pyomo'), default='core', help='Use subset of installation libraries, divided by source.') pipParser = subParsers.add_parser('pip', help='use pip as installer') @@ -611,6 +611,14 @@ def _readDependencies(initFile): actionArgs = '' addOptional = False limit = ['pip'] + elif args.subset == 'pyomo': + src = '' + installer = 'pyomo' + equals = '==' + equalsTail = '.*' + actionArgs = '' + addOptional = args.addOptional + limit = ['pyomo'] libs = getRequiredLibs(useOS=args.useOS, installMethod='conda', addOptional=addOptional, diff --git a/tests/framework/.gitignore b/tests/framework/.gitignore index 987d43a252..3c7248d198 100644 --- a/tests/framework/.gitignore +++ b/tests/framework/.gitignore @@ -139,6 +139,22 @@ TestXSD/test_more.xsd unit_tests/CustomDrivers/Basic unit_tests/Distributions/testDistrDump.pk user_guide/EnsembleModel/run_basic/sample/ +ROM/TimeSeries/ARMA/ARMAreseed/ +ROM/TimeSeries/ARMA/Segmented/arma.pk +ROM/TimeSeries/ARMA/Multicycle/arma.pk +DataObjects/StringIO/sample/ +CodeInterfaceTests/RAVEN/Code/sample/ +CodeInterfaceTests/Neutrino/Neutrino_Base/sampleNeutrino/[1-3] +CodeInterfaceTests/MOOSEBaseApps/MooseExample18WithGenericFile/myMC/[12] +CodeInterfaceTests/MOOSEBaseApps/MooseExample18OnlyGenericFile/myMC/[12] +CodeInterfaceTests/MOOSEBaseApps/MooseExample18OnlyGenericFile/myMC/ex18.i +CodeInterfaceTests/MOOSEBaseApps/MooseExample18WithGenericFile/myMC/ex18.i +CodeInterfaceTests/MOOSEBaseApps/InputParser/sample/[12]/formattest.i +CodeInterfaceTests/MOOSEBaseApps/InputParser/sample/[12]/out~formattest +CodeInterfaceTests/MOOSEBaseApps/InputParser/sample/formattest.i +hybridModel/logicalCode/logicalModelCode/ +Models/External/SerializePyomo/ +Models/External/SerializeWrkd/ user_guide/ForwardSamplingStrategies/RunDir/Grid/sample/ user_guide/ForwardSamplingStrategies/RunDir/MonteCarlo/sample/ user_guide/ForwardSamplingStrategies/RunDir/Stratified/sample/ diff --git a/tests/framework/Models/External/gold/GreyModelOutput_cyipopt.txt b/tests/framework/Models/External/gold/GreyModelOutput_cyipopt.txt new file mode 100644 index 0000000000..2401df74c4 --- /dev/null +++ b/tests/framework/Models/External/gold/GreyModelOutput_cyipopt.txt @@ -0,0 +1,51 @@ +---------------------< Output Summary >--------------------- + + +Problem: +- Name: unknown + Lower bound: -inf + Upper bound: 0.3587964554159516 + Number of objectives: 1 + Number of constraints: 1 + Number of variables: 3 + Number of binary variables: 0 + Number of integer variables: 0 + Number of continuous variables: 3 + Sense: minimize +Solver: +- Name: cyipopt + Status: ok + Return code: 0 + Message: b'Algorithm terminated successfully at a locally optimal point, satisfying the convergence tolerances (can be specified by options).' + Wallclock time: 0.14550156800000003 + Termination condition: optimal + +---------------------< Optimization Results >--------------------- + +1 Objective Declarations + obj : Size=1, Index=None, Active=True + Key : Active : Sense : Expression + None : True : minimize : egb.outputs[ans] + +1 ExternalGreyBoxBlock Declarations + egb : Size=1, Index=None, Active=True + 2 Set Declarations + _input_names_set : Size=1, Index=None, Ordered=Insertion + Key : Dimen : Domain : Size : Members + None : 1 : Any : 2 : {'y1', 'y2'} + _output_names_set : Size=1, Index=None, Ordered=Insertion + Key : Dimen : Domain : Size : Members + None : 1 : Any : 1 : {'ans',} + + 2 Var Declarations + inputs : Size=2, Index=egb._input_names_set + Key : Lower : Value : Upper : Fixed : Stale : Domain + y1 : 0.0 : 0.938770350603 : 1.0 : False : False : Reals + y2 : 0.0 : 0.938770350603 : 1.0 : False : False : Reals + outputs : Size=1, Index=egb._output_names_set + Key : Lower : Value : Upper : Fixed : Stale : Domain + ans : 0.358796465406 : 0.358796455416 : 0.975309912028 : False : False : Reals + + 4 Declarations: _input_names_set inputs _output_names_set outputs + +2 Declarations: egb obj diff --git a/tests/framework/Models/External/gold/SerializePyomo/rom_out.py b/tests/framework/Models/External/gold/SerializePyomo/rom_out.py new file mode 100644 index 0000000000..f7ffbf57e8 --- /dev/null +++ b/tests/framework/Models/External/gold/SerializePyomo/rom_out.py @@ -0,0 +1,118 @@ + +# MODEL GENERATED BY RAVEN (raven.inl.gov) +import pyomo.environ as pyo +from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock +from pyomo.contrib.pynumero.dependencies import (numpy as np) +from pyomo.contrib.pynumero.asl import AmplInterface +from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP +import os, sys, pickle +from contextlib import redirect_stdout +# RAVEN ROM PYOMO GREY MODEL CLASS + +class ravenROM(ExternalGreyBoxModel): + + def __init__(self, **kwargs): + self._rom_file = kwargs.get("rom_file") + self._raven_framework = kwargs.get("raven_framework") + self._order = kwargs.get("order") + self._raven_framework = os.path.abspath(self._raven_framework) + if not os.path.exists(self._raven_framework): + raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') + if os.path.isdir(os.path.join(self._raven_framework,"ravenframework")): + sys.path.append(self._raven_framework) + else: + raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') + from ravenframework.CustomDrivers import DriverUtils as dutils + dutils.doSetup() + # de-serialize the ROM + self._rom_file = os.path.abspath(self._rom_file) + if not os.path.exists(self._rom_file): + raise IOError('The serialized (binary) file has not been found in location "' + str(self._rom_file)+'" !') + self.rom = pickle.load(open(self._rom_file, mode='rb')) + # + self.settings = self.rom.getInitParams() + # get input names + self._input_names = self.settings.get('Features') + # n_inputs + self._n_inputs = len(self._input_names) + # get output names + self._output_names = self.settings.get('Target') + # n_outputs + self._n_outputs = len(self._output_names) + # input values storage + self._input_values = np.zeros(self._n_inputs, dtype=np.float64) + + def return_train_values(self, feat): + return self.rom.trainingSet.get(feat) + + def input_names(self): + return self._input_names + + def output_names(self): + return self._output_names + + def set_input_values(self, input_values): + assert len(input_values) == self._n_inputs + np.copyto(self._input_values, input_values) + + def evaluate_equality_constraints(self): + raise NotImplementedError('This method should not be called for this model.') + + def evaluate_outputs(self): + request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + outs = self.rom.evaluate(request) + eval_outputs = np.asarray([outs[k].flatten() for k in self._output_names], dtype=np.float64) + return eval_outputs.flatten() + + def evaluate_jacobian_outputs(self): + request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + derivatives = self.rom.derivatives(request, order=self._order) + jac = np.zeros((self._n_outputs, self._n_inputs)) + for tc, target in enumerate(self._output_names): + for fc, feature in enumerate(self._input_names): + jac[tc,fc] = derivatives['d{}|d{}'.format(target, feature)] + return jac + +def pyomoModel(ex_model): + m = pyo.ConcreteModel() + m.egb = ExternalGreyBoxBlock() + m.egb.set_external_model(ex_model) + for inp in ex_model.input_names(): + m.egb.inputs[inp].value = np.mean(ex_model.return_train_values(inp)) + m.egb.inputs[inp].setlb(np.min(ex_model.return_train_values(inp))) + m.egb.inputs[inp].setub(np.max(ex_model.return_train_values(inp))) + for out in ex_model.output_names(): + m.egb.outputs[out].value = np.mean(ex_model.return_train_values(out)) + m.egb.outputs[out].setlb(np.min(ex_model.return_train_values(out))) + m.egb.outputs[out].setub(np.max(ex_model.return_train_values(out))) + m.obj = pyo.Objective(expr=m.egb.outputs[out]) + + return m + +if __name__ == '__main__': + for cnt, item in enumerate(sys.argv): + if item.lower() == "-r": + rom_file = sys.argv[cnt+1] + if item.lower() == "-f": + raven_framework = sys.argv[cnt+1] + if item.lower() == "-order": + order = int(sys.argv[cnt+1]) + else: + order = 1 + ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework,'order':order}) + concreteModel = pyomoModel(ext_model) + ### here you should implement the optimization problem + ### + solver = pyo.SolverFactory('cyipopt') + solver.config.options['hessian_approximation'] = 'limited-memory' + results = solver.solve(concreteModel) + print(results) + count = 0 + lines = str(results).split("\n") + with open ("GreyModelOutput_cyipopt.txt", "w") as textfile: + with redirect_stdout(textfile): + print("---------------------< Output Summary >---------------------" + "\n") + for element in lines: + textfile.write(element + "\n") + print("---------------------< Optimization Results >---------------------" + "\n") + concreteModel.pprint() diff --git a/tests/framework/Models/External/gold/SerializePyomo/rom_pickle.pk b/tests/framework/Models/External/gold/SerializePyomo/rom_pickle.pk new file mode 100644 index 0000000000..5868202c16 Binary files /dev/null and b/tests/framework/Models/External/gold/SerializePyomo/rom_pickle.pk differ diff --git a/tests/framework/Models/External/rom_out.py b/tests/framework/Models/External/rom_out.py new file mode 100644 index 0000000000..ada7c169c9 --- /dev/null +++ b/tests/framework/Models/External/rom_out.py @@ -0,0 +1,113 @@ + +# MODEL GENERATED BY RAVEN (raven.inl.gov) +import pyomo.environ as pyo +from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock +from pyomo.contrib.pynumero.dependencies import (numpy as np) +from pyomo.contrib.pynumero.asl import AmplInterface +from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import CyIpoptSolver, CyIpoptNLP +import os, sys, pickle +from contextlib import redirect_stdout +# RAVEN ROM PYOMO GREY MODEL CLASS + +class ravenROM(ExternalGreyBoxModel): + + def __init__(self, **kwargs): + self._rom_file = kwargs.get("rom_file") + self._raven_framework = kwargs.get("raven_framework") + self._raven_framework = os.path.abspath(self._raven_framework) + if not os.path.exists(self._raven_framework): + raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') + if os.path.isdir(os.path.join(self._raven_framework,"ravenframework")): + sys.path.append(self._raven_framework) + else: + raise IOError('The RAVEN framework directory does not exist in location "' + str(self._raven_framework)+'" !') + from ravenframework.CustomDrivers import DriverUtils as dutils + dutils.doSetup() + # de-serialize the ROM + self._rom_file = os.path.abspath(self._rom_file) + if not os.path.exists(self._rom_file): + raise IOError('The serialized (binary) file has not been found in location "' + str(self._rom_file)+'" !') + self.rom = pickle.load(open(self._rom_file, mode='rb')) + # + self.settings = self.rom.getInitParams() + # get input names + self._input_names = self.settings.get('Features') + # n_inputs + self._n_inputs = len(self._input_names) + # get output names + self._output_names = self.settings.get('Target') + # n_outputs + self._n_outputs = len(self._output_names) + # input values storage + self._input_values = np.zeros(self._n_inputs, dtype=np.float64) + + def return_train_values(self, feat): + return self.rom.trainingSet.get(feat) + + def input_names(self): + return self._input_names + + def output_names(self): + return self._output_names + + def set_input_values(self, input_values): + assert len(input_values) == self._n_inputs + np.copyto(self._input_values, input_values) + + def evaluate_equality_constraints(self): + raise NotImplementedError('This method should not be called for this model.') + + def evaluate_outputs(self): + request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + outs = self.rom.evaluate(request) + eval_outputs = np.asarray([outs[k].flatten() for k in self._output_names], dtype=np.float64) + return eval_outputs.flatten() + + def evaluate_jacobian_outputs(self): + request = {k:np.asarray(v) for k,v in zip(self._input_names,self._input_values)} + derivatives = self.rom.derivatives(request) + jac = np.zeros((self._n_outputs, self._n_inputs)) + for tc, target in enumerate(self._output_names): + for fc, feature in enumerate(self._input_names): + jac[tc,fc] = derivatives['d{}|d{}'.format(target, feature)] + return jac + +def pyomoModel(ex_model): + m = pyo.ConcreteModel() + m.egb = ExternalGreyBoxBlock() + m.egb.set_external_model(ex_model) + for inp in ex_model.input_names(): + m.egb.inputs[inp].value = np.mean(ex_model.return_train_values(inp)) + m.egb.inputs[inp].setlb(np.min(ex_model.return_train_values(inp))) + m.egb.inputs[inp].setub(np.max(ex_model.return_train_values(inp))) + for out in ex_model.output_names(): + m.egb.outputs[out].value = np.mean(ex_model.return_train_values(out)) + m.egb.outputs[out].setlb(np.min(ex_model.return_train_values(out))) + m.egb.outputs[out].setub(np.max(ex_model.return_train_values(out))) + m.obj = pyo.Objective(expr=m.egb.outputs[out]) + + return m + +if __name__ == '__main__': + for cnt, item in enumerate(sys.argv): + if item.lower() == "-r": + rom_file = sys.argv[cnt+1] + if item.lower() == "-f": + raven_framework = sys.argv[cnt+1] + ext_model = ravenROM(**{'rom_file':rom_file,'raven_framework':raven_framework}) + concreteModel = pyomoModel(ext_model) + ### here you should implement the optimization problem + ### + solver = pyo.SolverFactory('cyipopt') + solver.config.options['hessian_approximation'] = 'limited-memory' + results = solver.solve(concreteModel) + print(results) + count = 0 + lines = str(results).split("\n") + with open ("GreyModelOutput_cyipopt.txt", "w") as textfile: + with redirect_stdout(textfile): + print("---------------------< Output Summary >---------------------" + "\n") + for element in lines: + textfile.write(element + "\n") + print("---------------------< Optimization Results >---------------------" + "\n") + concreteModel.pprint() diff --git a/tests/framework/Models/External/rom_pickle.pk b/tests/framework/Models/External/rom_pickle.pk new file mode 100644 index 0000000000..5868202c16 Binary files /dev/null and b/tests/framework/Models/External/rom_pickle.pk differ diff --git a/tests/framework/Models/External/serialize_pyomo.xml b/tests/framework/Models/External/serialize_pyomo.xml new file mode 100644 index 0000000000..a293999b10 --- /dev/null +++ b/tests/framework/Models/External/serialize_pyomo.xml @@ -0,0 +1,111 @@ + + + + framework.Models.External.serialize_pyomo + cogljj + 2021-08-18 + Models.ROM + + This test serializes a ROM to use with PYOMO. + It can be used with: + cd tests/framework/Models/External + python3 SerializePyomo/rom_out.py -r SerializePyomo/rom_pickle.pk -f ../../../.. + + + This test uses the analytic model ''attenuate``, which is documented in the analytical test documentation. + Additionally, values of the ''from`` variables are exactly determined because they are set in the extmod methods: + \begin{itemize} + \item fromReadMoreXML: pi (3.14159) + \item fromInit : sqrt(pi) (1.77245) + \item fromCNISelf : pi/2 (1.57080) + \item fromCNIDict : 2*sqrt(pi) (3.54491) + \end{itemize} + The exit strength ''ans`` is analytic with results as follows (Note that for testing purposes, the model always adds + 0.05 to \texttt{y2} before calculating the exit strength): + \begin{itemize} + \item \texttt{y1}, \texttt{y2}, \texttt{ans} + \item 0.0, 0.0, 0.97531 + \item 0.0, 1.0, 0.59156 + \item 1.0, 0.0, 0.59156 + \item 1.0, 1.0, 0.35880 + \end{itemize} + + + + + SerializePyomo + sample, train, serialize, pickle + + + + rom_out.py + rom_pickle.pk + + + + + dummyIN + attenuate + grid + samples + + + samples + out_rom + + + out_rom + rom_out.py + + + out_rom + rom_pickle.pk + + + + + + y1,y2 + ans,fromInit,fromReadMoreXML,fromCNISelf,fromCNIDict + + 3.14159 + + + + y1,y2 + ans + + + + + + 0 + 1 + + + + + + + dist + 0 1 + + + dist + 0 1 + + + + + + + y1,y2 + + + y1, y2 + ans + + + + + diff --git a/tests/framework/Models/External/tests b/tests/framework/Models/External/tests index 587c19b5ad..862b77c04c 100644 --- a/tests/framework/Models/External/tests +++ b/tests/framework/Models/External/tests @@ -9,6 +9,19 @@ input = 'serialize_ext_model_and_use.xml' csv = 'SerializeWrkd/samples_out.csv' [../] + [./pyomo_serialization] + type = 'RavenFramework' + input = 'serialize_pyomo.xml' + output = 'SerializePyomo/rom_out.py' + [../] + [./pynumero_GBM] + type = 'RavenPython' + input = 'rom_out.py -r rom_pickle.pk -f ../../../..' + output = 'GreyModelOutput_cyipopt.txt' + python3_only = True + minimum_library_versions = 'pyomo 6.4' + required_libraries = 'cyipopt' + [../] [./fmu_ext_model] type = 'RavenFramework' input = 'fmu_ext_model_and_use.xml'