Merge pull request #41 from Clonkk/devel

Devel

.gitignore

@@ -7,6 +7,7 @@ iteratorstest
 memleaktest
 arraymancertensortest
 conversionstest
+test_embedressources
 
 testresults/
 nimcache/

changelog.md

@@ -1,5 +1,20 @@
 Changelog for Nimjl. Date in format YYYY_MM_DD
 
+Release v0.7.0 - 2022_01_04
+===========================
+* Add Julia.useModule alias for jlUseModule
+* Add Julia.includeFile (include is reserved keyword) alias for jlInclude 
+* Add mechanism to embed julia files at compile-time and run the code at init for an easy way to distribute binary with Julia code contained
+* Add Pkg template to easily install new Julia package during init ; it is also compatible with the embedding stuff :
+* See ex09
+  ```nim
+    Julia.init: 
+      Pkg: 
+        add("LinearAlgebra")
+      Embed: 
+        file("myfile.jl")
+  ```
+
 Release v0.6.3 - 2021_11_05
 ===========================
 * Row major / col major handling when converting Arraymancer Tensor <-> JlArray

examples/ex05_module.nim

@@ -8,7 +8,6 @@ type
     z: string
 
 Julia.init() # Initialize Julia VM. This should be done once in the lifetime of your program.
-
 # Include Julia file
 jlInclude("localmodule.jl")
 # Use the module. If you're confused by the syntax, go and read through Julia's Manual where module usage is explained

examples/ex07_arrays.nim

@@ -3,6 +3,8 @@ import std/sugar
 import std/random
 import nimjl
 
+import arraymancer
+
 randomize()
 
 proc main() =
@@ -28,6 +30,11 @@ proc main() =
       let
         jlResArray = res.toJlArray(int64)
       echo "Sorted jlResArray=", jlResArray
+
+      # Or if you prefer using arraymancer Tensor
+      let
+        tensorEquivalent = res.to(Tensor[int64])
+      echo tensorEquivalent
     # End of Template equivalent to :
     # julia gc pop and julia gc push need to be in the same scope
     # julia_gc_pop()

examples/ex09_embed_file.nim

@@ -0,0 +1,42 @@
+import nimjl
+
+proc common() =
+  discard Julia.helloWorld1()
+  discard Julia.helloWorld2()
+
+  block:
+    let res = Julia.meanAB(12, 16)
+    echo res
+
+  block:
+    let res = Julia.squareDiv(9.3, 8.0)
+    echo res
+
+# All methods accomplish the same result
+# main_3 is the cleanest (subjective option) so it should be preferred
+
+proc main_2() =
+  # Manual embedding; must be done before init
+  jlEmbedDir("jlassets/")
+  jlEmbedFile("localasset.jl")
+
+  Julia.init()
+  defer: Julia.exit()
+
+  common()
+
+proc main_1() =
+  # Idiomatic way to embed Julia ressources and call them during after VM Init
+  Julia.init:
+    # Install package at init
+    Pkg:
+      add("LinearAlgebra")
+    Embed:
+      dir("jlassets/")
+      file("localasset.jl")
+  defer: Julia.exit()
+
+  common()
+
+when isMainModule:
+  main_1()

examples/jlassets/assets1.jl

@@ -0,0 +1,8 @@
+function helloWorld1()
+  println("HelloWorld1 from assets1")
+end
+
+function meanAB(a::Int, b::Int)
+  println("meanAB from assets1")
+  return (a+b)/2
+end

examples/localasset.jl

@@ -0,0 +1,8 @@
+function helloWorld2()
+  println("HelloWorld1 from localasset")
+end
+
+function squareDiv(a::Float64, b::Float64)
+  println("squareDiv from localasset")
+  return (a*a)/(b*b)
+end

nimjl.nimble

@@ -1,6 +1,6 @@
 # Nimjl
 # Licensed and distributed under MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
-version       = "0.6.3"
+version       = "0.7.0"
 author        = "Regis Caillaud"
 description   = "Nim Julia bridge"
 license       = "MIT"

nimjl/conversions/fromjl.nim

@@ -53,7 +53,8 @@ proc toNimVal[T](x: JlArray[T], locseq: var seq[T]) =
 
 proc toNimVal[T](x: JlValue, tensor: var Tensor[T]) =
   let x = toJlArray[T](x)
-  toNimVal(x, tensor)
+  # Assume rowMajor layout
+  toNimVal(x, tensor, rowMajor)
 
 proc toNimVal[T](x: JlValue, locseq: var seq[T]) =
   let x = toJlArray[T](x)
@@ -75,7 +76,7 @@ proc toNimVal(x: JlValue, jlfunc: var JlFunc) =
 {.pop.}
 
 # Public API
-proc to*[U](x: JlArray[U], T: typedesc, layout: static OrderType= rowMajor): T =
+proc to*[U](x: JlArray[U], T: typedesc, layout: static OrderType = rowMajor): T =
   when T is void:
     discard
   elif T is Tensor:

nimjl/cores.nim

@@ -1,40 +1,7 @@
 import ./types
 import ./private/jlcores
 import ./config
-import std/[strformat, os]
-
-proc jlVmIsInit*() : bool =
-  bool(jl_is_initialized())
-
-# Init & Exit function
-proc jlVmInit*() =
-  ## jlVmInit should only be called once per process
-  ## Subsequent calls after the first one will be ignored
-  if not jlVmIsInit():
-    jl_init()
-    return
-  # raise newException(JlError, "jl_init() must be called once per process")
-
-# Not exported for now because I don't know how it works
-proc jlVmInit(pathToImage: string) {.used.} =
-  ## Same as jlVmInit but with a pre-compiler image
-  if not jlVmIsInit():
-    let jlBinDir = cstring(JuliaPath / "bin")
-    jl_init_with_image(jlBinDir, pathToImage.cstring)
-    return
-  # raise newException(JlError, "jl_init_with_image(...) must be called once per process")
-
-proc jlVmSaveImage*(fname: string) =
-  jl_save_system_image(fname.cstring)
-
-proc jlVmExit*(exit_code: cint = 0.cint) =
-  ## jlVmExit should only be called once per process
-  ## Subsequent calls after the first one will be ignored
-  once:
-    jl_atexit_hook(exit_code)
-    return
-  # Do nothing -> atexit_hook must be called once
-  # raise newException(JlError, "jl_atexit_hook() must be called once per process")
+import std/[strformat, os, macros, tables]
 
 # Convert a string to Julia Symbol
 proc jlSym*(symname: string): JlSym =
@@ -83,3 +50,77 @@ proc jlGetModule*(modname: string): JlModule =
 
 # JlNothing is handy to have
 template JlNothing*(): JlValue = jlEval("nothing")
+
+template JlCode*(body: string) =
+  block:
+    discard jleval(body)
+
+proc jlVmIsInit*(): bool =
+  bool(jl_is_initialized())
+
+proc jlVmSaveImage*(fname: string) =
+  jl_save_system_image(fname.cstring)
+
+proc jlVmExit*(exit_code: cint = 0.cint) =
+  ## jlVmExit should only be called once per process
+  ## Subsequent calls after the first one will be ignored
+  once:
+    jl_atexit_hook(exit_code)
+    return
+  # Do nothing -> atexit_hook must be called once
+ # raise newException(JlError, "jl_atexit_hook() must be called once per process")
+
+#########################################
+var staticContents: Table[string, string]
+
+import std/logging
+
+proc loadJlRessources*() =
+  for key, content in staticContents.pairs():
+    info("> Nimjl loading Julia ressource: ", key, ".jl")
+    JlCode(content)
+
+# Init & Exit function
+proc jlVmInit*() =
+  ## jlVmInit should only be called once per process
+  ## Subsequent calls after the first one will be ignored
+  if not jlVmIsInit():
+    jl_init()
+    loadJlRessources()
+    return
+  # raise newException(JlError, "jl_init() must be called once per process")
+
+# Not exported for now because I don't know how it works
+proc jlVmInit(pathToImage: string) {.used.} =
+  ## Same as jlVmInit but with a pre-compiler image
+  if not jlVmIsInit():
+    let jlBinDir = cstring(JuliaPath / "bin")
+    jl_init_with_image(jlBinDir, pathToImage.cstring)
+    loadJlRessources()
+    return
+
+  # raise newException(JlError, "jl_init_with_image(...) must be called once per process")
+proc private_addKeyVal*(key, value: string) =
+  ## exported because macro doesn't work otherwise but shouldn't be used
+  staticContents[key] = value
+
+macro jlEmbedDir*(dirname: static[string]): untyped =
+  result = newStmtList()
+  let path = getProjectPath() / dirname
+  # echo path
+  # echo "------------------------------------------"
+
+  for file in path.walkDir:
+    if file.kind == pcFile:
+      let (dir, name, ext) = file.path.splitFile
+      if ext == ".jl":
+        # echo ">> ", name
+        let content = readFile(file.path)
+        result.add newCall("private_addKeyVal", newStrLitNode(name), newStrLitNode(content))
+
+  # echo "------------------------------------------"
+  # echo result.repr
+
+proc jlEmbedFile*(filename: static[string]) =
+  const jlContent = staticRead(getProjectPath() / filename)
+  staticContents[filename] = jlContent

nimjl/glucose.nim

@@ -12,9 +12,57 @@ type Julia* = object
 proc init*(jl: type Julia) =
   jlVmInit()
 
+template init*(jl: type Julia, body: untyped) =
+  # Pkg installation interface
+  var packages : seq[string]
+  template Pkg(innerbody: untyped) =
+    proc add(pkgname: string) =
+      packages.add pkgname
+    innerbody
+
+  # Embedding ressources interface
+  template Embed(innerbody: untyped) =
+    template file(filename: static[string]) =
+      jlEmbedFile(filename)
+
+    template dir(dirname: static[string]) =
+      jlEmbedDir(dirname)
+
+    template thisDir() =
+      jlEmbedDir("")
+
+    block:
+      innerbody
+
+  body
+
+  # Don't do anything if Julia is already initialized
+  if not jlVmIsInit():
+    jl_init()
+    # Module installation
+    Julia.useModule("Pkg")
+    let pkg = Julia.getModule("Pkg")
+    for pkgname in packages:
+      discard jlCall(pkg, "add", pkgname)
+
+    # Eval Julia code embedded
+    loadJlRessources()
+
+  else:
+    raise newException(JlError, "Error Julia.init() has already been initialized")
+
 proc exit*(jl: type Julia, exitcode: int = 0) =
   jlVmExit(exitcode.cint)
 
+proc useModule*(jl: type Julia, modname: string) =
+  jlUseModule(modname)
+
+proc getModule*(jl: type Julia, modname: string) : JlModule =
+  jlGetModule(modname)
+
+proc includeFile*(jl: type Julia, fname: string) =
+  jlInclude(fname)
+
 # macro loadModule*(jl: type Julia, modname: untyped) =
 # TODO generate a proc ``modname`` that returns module
 

readme.md

@@ -22,6 +22,8 @@ How to embed Julia w/ C :
 
 * https://github.com/JuliaLang/julia/tree/master/test/embedding
 
+* Read the Scinim getting-started [chapter on Nimjl](https://scinim.github.io/getting-started/external_language_integration/julia/basics.html)
+
 * ``legacy/`` folder contains previous experiment and examples of wrapping in C.
 
 * ``tests/testfull.nim`` is thet test suite
@@ -37,15 +39,13 @@ Julia is mostly oriented towards numerical computing so Arrays are THE most impo
 Mostly quality-of-life improvements, especially when handling arrays.
 
 * Improve Julia Arrays interop. from Nim.
-  * Array constructor API with most common proc
   * Supports complex Arrays
   * map / apply / reduce /fold
 
 ### Backlog
 
 * Support Julia chaining syntax
 * Add support for Enum types
-* Add a tag for tracing for Julia memory allocation
 
 ## Limitations
 
@@ -77,8 +77,31 @@ echo res # 2.0
 
 ```
 
-Take a look at ``tests/`` or ``examples/`` folder for typical examples.
+## New in version 0.7.0
+
+It is now possible to embed Julia files inside a Nim compiled binary to easily distribute Julia code. To make distribution possible, an API to call ``Pkg.add("...")`` has also been added.
 
+```nim
+import nimjl
+
+Julia.init:
+  Pkg:
+    add("DSP")
+    add("Wavelets")
+    add("LinearAlgebra")
+    add("Statistics")
+
+  Embed:
+    # embed all files with '*.jl' extension in folder ``JuliaToolBox/``
+    dir("JuliaToolBox/")
+    # embed all files with '*.jl' extension in the folder of he source file (at compilation) i.e. ``getProjectPath()`` 
+    thisDir()
+    # embed specific file; path should be relative to ``getProjectPath()``
+    file("localfile.jl")
+```
+See examples/ex09_embed_file.nim for a concrete example 
+
+Take a look at ``tests/`` or ``examples/`` folder for typical examples.
 
 # License
 

tests/assets/embed2.jl

@@ -0,0 +1,4 @@
+function testMe2()
+  println(">> Julia says... embedded function testMe2 exists !")
+  return true
+end

tests/embed.jl

@@ -0,0 +1,4 @@
+function testMe()
+  println(">> Julia says... embedded function testMe exists !")
+  return true
+end