Push package to PyPI and build the documentation automatically

.github/workflows/release-package.yml

@@ -8,78 +8,96 @@ on:
 
 jobs:
 
-  build:
+  docs:
     runs-on: ubuntu-latest
-
     steps:
+
       - name: Checkout repository
         uses: actions/checkout@v4
 
       - name: Install packages
         run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara
 
-      - name: Install poetry
-        run: pipx install poetry
-
       - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
 
-      - name: Install dependencies
-        run: poetry install --with docs
-
-      - name: Build package
-        run: poetry build
+      - name: Install poetry
+        run: python -m pip install poetry
 
-      - name: Build documentation
+      - name: Build docs
         run: make generate_docs
 
       - name: Setup pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v5
 
       - name: Upload documentation artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: docs/
 
-      - name: Upload package artifcat
-        uses: actions/upload-artifact@v3
+  deploy-gh-pages:
+      runs-on: ubuntu-latest
+      needs: docs
+
+      permissions:
+        pages: write
+        id-token: write
+
+      environment:
+        name: github-pages
+        url: ${{ steps.deployment.outputs.page_url }}
+
+      steps:
+        - name: Deploy to GitHub Pages
+          id: deployment
+          uses: actions/deploy-pages@v2
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install packages
+        run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Install poetry
+        run: python -m pip install poetry
+
+      - name: Install dependencies
+        run: poetry install --with unstable
+
+      - name: Build package
+        run: poetry build
+
+      - name: Upload package artifact
+        uses: actions/upload-artifact@v4
         with:
           name: python-package-distributions
           path: dist/
 
+  publish-to-pypi:
+    runs-on: ubuntu-latest
+    needs: build
 
-#   publish-to-pypi:
-#     runs-on: ubuntu-latest
-#     needs: build
-# 
-#     steps:
-#       - name: Download dist files
-#         uses: actions/download-artifact@v3
-#         with:
-#           name: python-package-distributions
-#           path: dist/
-# 
-#       - name: Publish to PyPI
-#         uses: pypa/gh-action-pypi-publish@release/v1
-#         # with:
-#         #   repository-url: https://test.pypi.org/legacy/
-# 
-# 
-#   deploy-gh-pages:
-#     runs-on: ubuntu-latest
-#     needs: build
-# 
-#     permissions:
-#       pages: write
-#       id-token: write
-# 
-#     environment:
-#       name: github-pages
-#       url: ${{ steps.deployment.outputs.page_url }}
-# 
-#     steps:
-#       - name: Deploy to GitHub Pages
-#         id: deployment
-#         uses: actions/deploy-pages@v2
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Download dist files
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+         skip-existing: true

.github/workflows/test-package.yml

@@ -12,7 +12,6 @@ jobs:
 
   docs:
     runs-on: ubuntu-latest
-
     steps:
 
       - name: Checkout repository
@@ -21,17 +20,13 @@ jobs:
       - name: Install packages
         run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara
 
-      - name: Install poetry
-        run: pipx install poetry
-
       - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
-          cache: 'poetry'
 
-      - name: Install dependencies
-        run: poetry install --with docs
+      - name: Install poetry
+        run: python -m pip install poetry
 
       - name: Build docs
         run: make generate_docs
@@ -51,18 +46,17 @@ jobs:
 
       - name: Install packages
         run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara
-
-      - name: Install poetry
-        run: pipx install poetry
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
+
+      - name: Install poetry
+        run: python -m pip install poetry
 
       - name: Install dependencies
-        run: poetry install --with test
+        run: poetry install --with test,unstable
 
       - name: Build package
         run: poetry build
@@ -90,3 +84,30 @@ jobs:
       - name: Show error log
         if: always()
         run: cat error.log
+
+      - name: Upload package artifcat
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-to-test-pypi:
+    runs-on: ubuntu-latest
+    needs: test
+
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Download dist files
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+    - name: Publish to Test PyPI
+      if: ${{ github.event_name == 'push' }}
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+         skip-existing: true
+         repository-url: https://test.pypi.org/legacy/

Makefile

@@ -17,7 +17,7 @@ USE_DOCKER ?=
 
 prepare_docs:
 	@echo "Preparing documentation."
-	poetry install --only docs
+	poetry install --with docs,unstable
 	poetry run python $(DOCS_SRC_DIR)/generate_documentation.py
 	mkdir -p $(DOCS_DIST_DIR)/logos
 	mkdir -p $(DOCS_DIST_DIR)/expansion/logos

README.md

@@ -1,6 +1,6 @@
 # MISP modules
 
-[![Build status](https://github.com/MISP/misp-modules/actions/workflows/python-package.yml/badge.svg)](https://github.com/MISP/misp-modules/actions/workflows/python-package.yml)[![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=main)](https://coveralls.io/github/MISP/misp-modules?branch=main)
+[![Build status](https://github.com/MISP/misp-modules/actions/workflows/test-package.yml/badge.svg)](https://github.com/MISP/misp-modules/actions/workflows/test-package.yml)[![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=main)](https://coveralls.io/github/MISP/misp-modules?branch=main)
 [![codecov](https://codecov.io/gh/MISP/misp-modules/branch/main/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules)
 
 MISP modules are autonomous modules that can be used to extend [MISP](https://github.com/MISP/MISP) for new services such as expansion, import, export and workflow action.

docs/install.md

@@ -8,7 +8,24 @@ Once the virtual environment is loaded just use the command:
 pip install misp-modules
 ~~~~
 
-Note: this install method might not yet be available.
+Note that the dependencies will require a number of system packages installed. On Ubuntu these packages are `libpoppler-cpp-dev`, `libzbar0`, and `tesseract-ocr`. For an updated list, check the github action used to test the build inside `.github/workflows`.
+
+Because PyPI does not support git for direct dependencies, the following packages will not be installed by default `otdreader`, `google-search-api`, `trustar`, `pydnstrails`, `pyonyphe`. You can either install them manually or let the modules depending on them gracefully fail.
+
+~~~~bash
+pip install \
+	git+https://github.com/cartertemm/ODTReader.git \
+	git+https://github.com/abenassi/Google-Search-API \
+	git+https://github.com/SteveClement/trustar-python.git \
+	git+https://github.com/sebdraven/pydnstrails.git \
+	git+https://github.com/sebdraven/pyonyphe.git
+~~~~
+
+You can now run `misp-modules` by invoking it (you might need to reload the virtual environment to update the search path used for executables).
+
+~~~~bash
+misp-modules
+~~~~
 
 
 ## Install from cloned repository
@@ -20,9 +37,11 @@ Once `poetry` is installed, you can clone the repository and install `misp-modul
 ~~~~bash
 git clone https://github.com/MISP/misp-modules.git && cd misp-modules
 git submodule update --init
-poetry install
+poetry install --with unstable
 ~~~~
 
+The switch `--with unstable` will also install dependencies available only on `git` repositories (which are manually installed when using pip).
+
 Note that the dependencies will require a number of system packages installed. On Ubuntu these packages are `libpoppler-cpp-dev`, `libzbar0`, and `tesseract-ocr`. For an updated list, check the github action used to test the build inside `.github/workflows`.
 
 
@@ -58,7 +77,7 @@ systemctl enable --now misp-modules
 To run tests you need to install misp-modules from the cloned repository, run the server, and then run the tests. You can do all these step with `poetry`.
 
 ~~~~bash
-poetry install
+poetry install --with unstable
 poetry run misp-modules
 ~~~~
 
@@ -123,7 +142,7 @@ You have two choices, the first approach uses `poetry export` to export the enti
 This is quite straightforward but it assumes your target system is relatively similar (same distribution, architecture, libaries).
 
 ~~~~bash
-poetry install
+poetry install --with unstable
 poetry self add poetry-plugin-bundle
 poetry bundle venv /destination/path/
 ~~~~
@@ -146,7 +165,7 @@ Then, run the following commands to generate your very own `requirements.txt`.
 
 ~~~~bash
 poetry lock
-poetry install
+poetry install --with unstable
 poetry self add poetry-plugin-export
 poetry export --without-hashes -f requirements.txt -o requirements.txt
 ~~~~