ci: updated workflow

.github/workflows/docker-crawlab.yml

@@ -26,6 +26,7 @@ env:
 
 jobs:
   setup:
+    name: Setup
     runs-on: ubuntu-latest
     outputs:
       backend_changed: ${{ steps.check_changed_files.outputs.backend_changed }}
@@ -112,6 +113,7 @@ jobs:
           fi
 
   build_base_image:
+    name: Build base image
     needs: [ setup ]
     if: needs.setup.outputs.base_image_changed == 'true' || needs.setup.outputs.workflow_changed == 'true'
     runs-on: ubuntu-latest
@@ -149,15 +151,18 @@ jobs:
         image: mongo:5
         ports:
           - 27017:27017
+    strategy:
+      matrix:
+        package: [core, db]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: 'core/go.mod'
-          cache-dependency-path: 'core/go.sum'
+          go-version-file: '${{ matrix.package }}/go.mod'
+          cache-dependency-path: '${{ matrix.package }}/go.sum'
       - name: Run tests
-        working-directory: core
+        working-directory: ${{ matrix.package }}
         run: |
           # Find all directories containing *_test.go files
           test_dirs=$(find . -name "*_test.go" -exec dirname {} \; | sort -u)
@@ -275,6 +280,7 @@ jobs:
             ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }}
 
   test_crawlab:
+    name: Test crawlab
     needs: [setup, build_crawlab]
     if: ${{ always() && needs.build_crawlab.result == 'success' }}
     runs-on: ubuntu-latest
@@ -332,6 +338,7 @@ jobs:
           destination_dir: playwright-report/${{ needs.setup.outputs.version }}
 
   push_images:
+    name: Push images
     if: ${{ always() && needs.test_crawlab.result == 'success' }}
     needs: [setup, test_crawlab]
     runs-on: ubuntu-latest

db/mongo/col.go

@@ -3,7 +3,8 @@ package mongo
 import (
 	"context"
 	"errors"
-	"github.com/crawlab-team/crawlab/trace"
+
+	"github.com/apex/log"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -53,26 +54,31 @@ type Col struct {
 func (col *Col) Insert(doc interface{}) (id primitive.ObjectID, err error) {
 	res, err := col.c.InsertOne(col.ctx, doc)
 	if err != nil {
-		return primitive.NilObjectID, trace.TraceError(err)
+		log.Errorf("error inserting document: %v", err)
+		return primitive.NilObjectID, err
 	}
 	if id, ok := res.InsertedID.(primitive.ObjectID); ok {
 		return id, nil
 	}
-	return primitive.NilObjectID, trace.TraceError(errors.New("InsertedID is not ObjectID"))
+	err = errors.New("InsertedID is not ObjectID")
+	log.Errorf("error inserting document: %v", err)
+	return primitive.NilObjectID, err
 }
 
 func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err error) {
 	res, err := col.c.InsertMany(col.ctx, docs)
 	if err != nil {
-		return nil, trace.TraceError(err)
+		return nil, err
 	}
 	for _, v := range res.InsertedIDs {
 		switch v.(type) {
 		case primitive.ObjectID:
 			id := v.(primitive.ObjectID)
 			ids = append(ids, id)
 		default:
-			return nil, trace.TraceError(errors.New("InsertedID is not ObjectID"))
+			err = errors.New("InsertedID is not ObjectID")
+			log.Errorf("error inserting document: %v", err)
+			return nil, err
 		}
 	}
 	return ids, nil
@@ -81,7 +87,8 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er
 func (col *Col) UpdateId(id primitive.ObjectID, update interface{}) (err error) {
 	_, err = col.c.UpdateOne(col.ctx, bson.M{"_id": id}, update)
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error updating document: %v", err)
+		return err
 	}
 	return nil
 }
@@ -97,7 +104,8 @@ func (col *Col) UpdateWithOptions(query bson.M, update interface{}, opts *option
 		_, err = col.c.UpdateMany(col.ctx, query, update, opts)
 	}
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error updating document: %v", err)
+		return err
 	}
 	return nil
 }
@@ -117,15 +125,17 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options.
 		_, err = col.c.ReplaceOne(col.ctx, query, doc, opts)
 	}
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error replacing document: %v", err)
+		return err
 	}
 	return nil
 }
 
 func (col *Col) DeleteId(id primitive.ObjectID) (err error) {
 	_, err = col.c.DeleteOne(col.ctx, bson.M{"_id": id})
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error deleting document: %v", err)
+		return err
 	}
 	return nil
 }
@@ -141,7 +151,8 @@ func (col *Col) DeleteWithOptions(query bson.M, opts *options.DeleteOptions) (er
 		_, err = col.c.DeleteMany(col.ctx, query, opts)
 	}
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error deleting document: %v", err)
+		return err
 	}
 	return nil
 }
@@ -223,15 +234,17 @@ func (col *Col) Aggregate(pipeline mongo.Pipeline, opts *options.AggregateOption
 func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) {
 	_, err = col.c.Indexes().CreateOne(col.ctx, indexModel)
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error creating index: %v", err)
+		return err
 	}
 	return nil
 }
 
 func (col *Col) CreateIndexes(indexModels []mongo.IndexModel) (err error) {
 	_, err = col.c.Indexes().CreateMany(col.ctx, indexModels)
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error creating indexes: %v", err)
+		return err
 	}
 	return nil
 }
@@ -247,25 +260,29 @@ func (col *Col) MustCreateIndexes(indexModels []mongo.IndexModel) {
 func (col *Col) DeleteIndex(name string) (err error) {
 	_, err = col.c.Indexes().DropOne(col.ctx, name)
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error deleting index: %v", err)
+		return err
 	}
 	return nil
 }
 
 func (col *Col) DeleteAllIndexes() (err error) {
 	_, err = col.c.Indexes().DropAll(col.ctx)
 	if err != nil {
-		return trace.TraceError(err)
+		log.Errorf("error deleting all indexes: %v", err)
+		return err
 	}
 	return nil
 }
 
 func (col *Col) ListIndexes() (indexes []map[string]interface{}, err error) {
 	cur, err := col.c.Indexes().List(col.ctx)
 	if err != nil {
+		log.Errorf("error listing indexes: %v", err)
 		return nil, err
 	}
 	if err := cur.All(col.ctx, &indexes); err != nil {
+		log.Errorf("error listing indexes: %v", err)
 		return nil, err
 	}
 	return indexes, nil

db/mongo/col_test.go

@@ -59,7 +59,8 @@ func TestGetMongoColWithDb(t *testing.T) {
 	dbName := "test_db"
 	colName := "test_col"
 
-	col := GetMongoColWithDb(colName, dbName)
+	db := GetMongoDb(dbName)
+	col := GetMongoColWithDb(colName, db)
 	require.Equal(t, colName, col.c.Name())
 	require.Equal(t, dbName, col.db.Name())
 }

db/mongo/db.go

@@ -1,35 +1,24 @@
 package mongo
 
 import (
-	"github.com/crawlab-team/crawlab/trace"
+	"github.com/apex/log"
 	"github.com/spf13/viper"
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
-func GetMongoDb(dbName string, opts ...DbOption) (db *mongo.Database) {
+func GetMongoDb(dbName string) *mongo.Database {
+	// Use default database name if not provided
 	if dbName == "" {
-		dbName = viper.GetString("mongo.db")
-	}
-	if dbName == "" {
-		dbName = "test"
-	}
-
-	_opts := &DbOptions{}
-	for _, op := range opts {
-		op(_opts)
+		if dbName = viper.GetString("mongo.db"); dbName == "" {
+			dbName = "test"
+		}
 	}
 
-	var c *mongo.Client
-	if _opts.client == nil {
-		var err error
-		c, err = GetMongoClient()
-		if err != nil {
-			trace.PrintError(err)
-			return nil
-		}
-	} else {
-		c = _opts.client
+	c, err := GetMongoClient()
+	if err != nil {
+		log.Errorf("error getting mongo client: %v", err)
+		return nil
 	}
 
-	return c.Database(dbName, nil)
+	return c.Database(dbName)
 }

docker/base-image/install/go/go.sh

@@ -19,6 +19,10 @@ eval "$(goenv init -)"
 goenv install ${version}
 goenv global ${version}
 
+# Create symbolic links
+ln -sf "$(goenv which go)" /usr/local/bin/go
+ln -sf "$(goenv which gofmt)" /usr/local/bin/gofmt
+
 # verify
 go_version=$(go version)
 if [[ $go_version =~ "go${version}" ]]; then

docker/base-image/install/java/java.sh

@@ -1,10 +1,35 @@
 #!/bin/bash
 
-# Exit on error
-set -e
-
-# Update package list and install OpenJDK 11
-DEBIAN_FRONTEND=noninteractive apt-get update && \
-    apt-get install -y --no-install-recommends openjdk-11-jdk && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+version="11.0.12-open"
+
+# Install SDKMAN!
+curl -s "https://get.sdkman.io" | bash
+
+# Source SDKMAN!
+source "$HOME/.sdkman/bin/sdkman-init.sh"
+
+# Install Java 11 (you can specify vendor, e.g., 11.0.12-open for OpenJDK)
+sdk install java ${version}
+
+# Set Java 11 as default
+sdk default java ${version}
+
+# Create symbolic links
+ln -sf "$(sdkman which java)" /usr/local/bin/java
+ln -sf "$(sdkman which javac)" /usr/local/bin/javac
+
+# Verify
+java_version=$(java -version)
+if [[ $java_version =~ "${version}" ]]; then
+	:
+else
+	echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\""
+	exit 1
+fi
+javac_version=$(javac -version)
+if [[ $javac_version =~ "${version}" ]]; then
+	:
+else
+	echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\""
+	exit 1
+fi

docker/base-image/install/node/node.sh

@@ -22,6 +22,12 @@ nvm install ${version}
 nvm use ${version}
 nvm alias default ${version}
 
+# Create symbolic links
+ln -sf "$(nvm which node)" /usr/local/bin/node
+ln -sf "$(nvm which npm)" /usr/local/bin/npm
+ln -sf "$(nvm which yarn)" /usr/local/bin/yarn
+ln -sf "$(nvm which pnpm)" /usr/local/bin/pnpm
+
 # verifies the right Node.js version is in the environment
 if [[ ! "$(node -v)" =~ ^v${version} ]]; then
 	echo "Node.js version is not v${version}.x"

docker/base-image/install/python/python.sh

@@ -41,6 +41,10 @@ eval "$(pyenv virtualenv-init -)"
 pyenv install ${version}
 pyenv global ${version}
 
+# Create symbolic links
+ln -sf $(pyenv which python) /usr/local/bin/python
+ln -sf $(pyenv which pip) /usr/local/bin/pip
+
 # verify
 python_version=$(python -V)
 if [[ $python_version =~ "Python ${version}" ]]; then