[ update ]: Cythonize Machine Learning Utilities

.github/workflows/gru_pipeline.yaml

@@ -127,6 +127,15 @@ jobs:
         with:
           name: datasets
 
+      - name: Install Cython and Build ML Module
+        run: |
+          apt-get update && \
+            apt-get install -y gcc python3-dev gnupg
+
+          pip install cython
+          cd training && \
+            python setup.py build_ext --inplace && cd ..
+
       - name: Modeling and Training
         # if: env.match != 'true'
         run: |
@@ -136,7 +145,7 @@ jobs:
           mkdir pickles
           mkdir posttrained
 
-          python training.py \
+          python trainingcli.py \
             --epochs=200     \
             --batchs=32      \
             --sequences=5    \
@@ -568,9 +577,9 @@ jobs:
           sleep 60
           chmod +x endpoints_test.sh && ./endpoints_test.sh
 
-      - name: Set Pipeline Schedule
-        # if: env.match != 'true'
-        run: echo "$(date +'%Y-%m-%d')" > schedulers/gru_schedule.ctl
+      # - name: Set Pipeline Schedule
+      #   if: env.match != 'true'
+      #   run: echo "$(date +'%Y-%m-%d')" > schedulers/gru_schedule.ctl
 
       # - name: Commit changes
       #   if: env.match != 'true'

.github/workflows/lstm_gru_pipeline.yaml

@@ -127,6 +127,15 @@ jobs:
         with:
           name: datasets
 
+      - name: Install Cython and Build ML Module
+        run: |
+          apt-get update && \
+            apt-get install -y gcc python3-dev gnupg
+
+          pip install cython
+          cd training && \
+            python setup.py build_ext --inplace && cd ..
+
       - name: Modeling and Training
         # if: env.match != 'true'
         run: |
@@ -136,15 +145,15 @@ jobs:
           mkdir pickles
           mkdir posttrained
 
-          python training.py \
+          python trainingcli.py \
             --epochs=200     \
             --batchs=32      \
             --sequences=5    \
             --algorithm=LSTM_GRU
 
-      - name: Set Pipeline Schedule
-        # if: env.match != 'true'
-        run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_gru_schedule.ctl
+      # - name: Set Pipeline Schedule
+      #   if: env.match != 'true'
+      #   run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_gru_schedule.ctl
 
       - name: Zip Posttrained, Models, and Pickles
         # if: env.match != 'true'

.github/workflows/lstm_pipeline.yaml

@@ -126,6 +126,15 @@ jobs:
         with:
           name: datasets
 
+      - name: Install Cython and Build ML Module
+        run: |
+          apt-get update && \
+            apt-get install -y gcc python3-dev gnupg
+
+          pip install cython
+          cd training && \
+            python setup.py build_ext --inplace && cd ..
+
       - name: Modeling and Training
         # if: env.match != 'true'
         run: |
@@ -135,15 +144,15 @@ jobs:
           mkdir pickles
           mkdir posttrained
 
-          python training.py \
+          python trainingcli.py \
             --epochs=200     \
             --batchs=32      \
             --sequences=5    \
             --algorithm=LSTM
 
-      - name: Set Pipeline Schedule
-        # if: env.match != 'true'
-        run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl
+      # - name: Set Pipeline Schedule
+      #   if: env.match != 'true'
+      #   run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl
 
       - name: Zip Posttrained, Models, and Pickles
         # if: env.match != 'true'

Makefile

@@ -1,5 +1,56 @@
+# makefile for building and running the application
+
+.PHONY: all cutils ctrain cpostman converter run clean
+
+# default target to build all components
+all: cutils ctrain cpostman
+
+# build Cython extensions in restful/cutils
 cutils:
-	cd restful/cutils && python setup.py build_ext --inplace && cd ../..
+	@echo "Building Cython extensions in restful/cutils..."
+	@cd restful/cutils && python setup.py build_ext --inplace
+
+
+# build Cython extensions in training
+ctrain:
+	@echo "Building Cython extensions in training..."
+	@cd training && python setup.py build_ext --inplace
+
+
+# build Cython extensions in postman
+cpostman:
+	@echo "Building Cython extensions in postman..."
+	@cd postman && python setup.py build_ext --inplace
 
+
+# run the converter script
+converter:
+	@echo "Running converter script..."
+	@python postman/converter.py
+
+
+# run the application with uvicorn
 run:
-	uvicorn app:app --host 0.0.0.0 --port 7860 --reload
+	@echo "Starting the application..."
+	@uvicorn app:app --host 0.0.0.0 --port 7860 --reload
+
+
+# clean up build artifacts
+clean:
+	@echo "Cleaning up build artifacts..."
+	@find . -type f -name "*.so" -delete
+	@find . -type f -name "*.c" -delete
+	@find . -type f -name "*.cpp" -delete
+	@find . -type d -name "__pycache__" -exec rm -r {} +
+
+
+# help message
+help:
+	@echo "Usage:"
+	@echo "  make all       - Build all Cython extensions"
+	@echo "  make cutils    - Build Cython extensions in restful/cutils"
+	@echo "  make ctrain    - Build Cython extensions in training"
+	@echo "  make cpostman  - Build Cython extensions in postman"
+	@echo "  make converter - Run the converter script"
+	@echo "  make run       - Start the application with Uvicorn"
+	@echo "  make clean     - Remove build artifacts"

converter.py → postman/converter.py

@@ -1,5 +1,6 @@
 import json
 
+# Json Processor
 class JSONProcessor:
     def __init__(self, input_file: str, output_file: str) -> None:
         self.input_file:  str = input_file
@@ -30,5 +31,4 @@ def main():
     symbols = processor.extract_symbols()
     processor.save_json(symbols)
 
-
-if __name__ == "__main__": main()
+if __name__ == "__main__": main()

postman/json.c

@@ -0,0 +1 @@
+#error Do not use this file, it is the result of a failed Cython compilation.