diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index b72b5c79..0e3a4d1b 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -26,6 +26,7 @@ env: jobs: setup: + name: Setup runs-on: ubuntu-latest outputs: backend_changed: ${{ steps.check_changed_files.outputs.backend_changed }} @@ -112,6 +113,7 @@ jobs: fi build_base_image: + name: Build base image needs: [ setup ] if: needs.setup.outputs.base_image_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' runs-on: ubuntu-latest @@ -149,15 +151,18 @@ jobs: image: mongo:5 ports: - 27017:27017 + strategy: + matrix: + package: [core, db] steps: - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: 'core/go.mod' - cache-dependency-path: 'core/go.sum' + go-version-file: '${{ matrix.package }}/go.mod' + cache-dependency-path: '${{ matrix.package }}/go.sum' - name: Run tests - working-directory: core + working-directory: ${{ matrix.package }} run: | # Find all directories containing *_test.go files test_dirs=$(find . -name "*_test.go" -exec dirname {} \; | sort -u) @@ -275,6 +280,7 @@ jobs: ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }} test_crawlab: + name: Test crawlab needs: [setup, build_crawlab] if: ${{ always() && needs.build_crawlab.result == 'success' }} runs-on: ubuntu-latest @@ -332,6 +338,7 @@ jobs: destination_dir: playwright-report/${{ needs.setup.outputs.version }} push_images: + name: Push images if: ${{ always() && needs.test_crawlab.result == 'success' }} needs: [setup, test_crawlab] runs-on: ubuntu-latest diff --git a/db/mongo/col.go b/db/mongo/col.go index 56b91a5f..925270f0 100644 --- a/db/mongo/col.go +++ b/db/mongo/col.go @@ -3,7 +3,8 @@ package mongo import ( "context" "errors" - "github.com/crawlab-team/crawlab/trace" + + "github.com/apex/log" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -53,18 +54,21 @@ type Col struct { func (col *Col) Insert(doc interface{}) (id primitive.ObjectID, err error) { res, err := col.c.InsertOne(col.ctx, doc) if err != nil { - return primitive.NilObjectID, trace.TraceError(err) + log.Errorf("error inserting document: %v", err) + return primitive.NilObjectID, err } if id, ok := res.InsertedID.(primitive.ObjectID); ok { return id, nil } - return primitive.NilObjectID, trace.TraceError(errors.New("InsertedID is not ObjectID")) + err = errors.New("InsertedID is not ObjectID") + log.Errorf("error inserting document: %v", err) + return primitive.NilObjectID, err } func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err error) { res, err := col.c.InsertMany(col.ctx, docs) if err != nil { - return nil, trace.TraceError(err) + return nil, err } for _, v := range res.InsertedIDs { switch v.(type) { @@ -72,7 +76,9 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er id := v.(primitive.ObjectID) ids = append(ids, id) default: - return nil, trace.TraceError(errors.New("InsertedID is not ObjectID")) + err = errors.New("InsertedID is not ObjectID") + log.Errorf("error inserting document: %v", err) + return nil, err } } return ids, nil @@ -81,7 +87,8 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er func (col *Col) UpdateId(id primitive.ObjectID, update interface{}) (err error) { _, err = col.c.UpdateOne(col.ctx, bson.M{"_id": id}, update) if err != nil { - return trace.TraceError(err) + log.Errorf("error updating document: %v", err) + return err } return nil } @@ -97,7 +104,8 @@ func (col *Col) UpdateWithOptions(query bson.M, update interface{}, opts *option _, err = col.c.UpdateMany(col.ctx, query, update, opts) } if err != nil { - return trace.TraceError(err) + log.Errorf("error updating document: %v", err) + return err } return nil } @@ -117,7 +125,8 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options. _, err = col.c.ReplaceOne(col.ctx, query, doc, opts) } if err != nil { - return trace.TraceError(err) + log.Errorf("error replacing document: %v", err) + return err } return nil } @@ -125,7 +134,8 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options. func (col *Col) DeleteId(id primitive.ObjectID) (err error) { _, err = col.c.DeleteOne(col.ctx, bson.M{"_id": id}) if err != nil { - return trace.TraceError(err) + log.Errorf("error deleting document: %v", err) + return err } return nil } @@ -141,7 +151,8 @@ func (col *Col) DeleteWithOptions(query bson.M, opts *options.DeleteOptions) (er _, err = col.c.DeleteMany(col.ctx, query, opts) } if err != nil { - return trace.TraceError(err) + log.Errorf("error deleting document: %v", err) + return err } return nil } @@ -223,7 +234,8 @@ func (col *Col) Aggregate(pipeline mongo.Pipeline, opts *options.AggregateOption func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) { _, err = col.c.Indexes().CreateOne(col.ctx, indexModel) if err != nil { - return trace.TraceError(err) + log.Errorf("error creating index: %v", err) + return err } return nil } @@ -231,7 +243,8 @@ func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) { func (col *Col) CreateIndexes(indexModels []mongo.IndexModel) (err error) { _, err = col.c.Indexes().CreateMany(col.ctx, indexModels) if err != nil { - return trace.TraceError(err) + log.Errorf("error creating indexes: %v", err) + return err } return nil } @@ -247,7 +260,8 @@ func (col *Col) MustCreateIndexes(indexModels []mongo.IndexModel) { func (col *Col) DeleteIndex(name string) (err error) { _, err = col.c.Indexes().DropOne(col.ctx, name) if err != nil { - return trace.TraceError(err) + log.Errorf("error deleting index: %v", err) + return err } return nil } @@ -255,7 +269,8 @@ func (col *Col) DeleteIndex(name string) (err error) { func (col *Col) DeleteAllIndexes() (err error) { _, err = col.c.Indexes().DropAll(col.ctx) if err != nil { - return trace.TraceError(err) + log.Errorf("error deleting all indexes: %v", err) + return err } return nil } @@ -263,9 +278,11 @@ func (col *Col) DeleteAllIndexes() (err error) { func (col *Col) ListIndexes() (indexes []map[string]interface{}, err error) { cur, err := col.c.Indexes().List(col.ctx) if err != nil { + log.Errorf("error listing indexes: %v", err) return nil, err } if err := cur.All(col.ctx, &indexes); err != nil { + log.Errorf("error listing indexes: %v", err) return nil, err } return indexes, nil diff --git a/db/mongo/col_test.go b/db/mongo/col_test.go index 4d8533d3..d9f1462e 100644 --- a/db/mongo/col_test.go +++ b/db/mongo/col_test.go @@ -59,7 +59,8 @@ func TestGetMongoColWithDb(t *testing.T) { dbName := "test_db" colName := "test_col" - col := GetMongoColWithDb(colName, dbName) + db := GetMongoDb(dbName) + col := GetMongoColWithDb(colName, db) require.Equal(t, colName, col.c.Name()) require.Equal(t, dbName, col.db.Name()) } diff --git a/db/mongo/db.go b/db/mongo/db.go index 1b29790d..3f30d620 100644 --- a/db/mongo/db.go +++ b/db/mongo/db.go @@ -1,35 +1,24 @@ package mongo import ( - "github.com/crawlab-team/crawlab/trace" + "github.com/apex/log" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/mongo" ) -func GetMongoDb(dbName string, opts ...DbOption) (db *mongo.Database) { +func GetMongoDb(dbName string) *mongo.Database { + // Use default database name if not provided if dbName == "" { - dbName = viper.GetString("mongo.db") - } - if dbName == "" { - dbName = "test" - } - - _opts := &DbOptions{} - for _, op := range opts { - op(_opts) + if dbName = viper.GetString("mongo.db"); dbName == "" { + dbName = "test" + } } - var c *mongo.Client - if _opts.client == nil { - var err error - c, err = GetMongoClient() - if err != nil { - trace.PrintError(err) - return nil - } - } else { - c = _opts.client + c, err := GetMongoClient() + if err != nil { + log.Errorf("error getting mongo client: %v", err) + return nil } - return c.Database(dbName, nil) + return c.Database(dbName) } diff --git a/db/mongo/db_options.go b/db/mongo/db_options.go deleted file mode 100644 index 480cc4c6..00000000 --- a/db/mongo/db_options.go +++ /dev/null @@ -1,15 +0,0 @@ -package mongo - -import "go.mongodb.org/mongo-driver/mongo" - -type DbOption func(options *DbOptions) - -type DbOptions struct { - client *mongo.Client -} - -func WithDbClient(c *mongo.Client) DbOption { - return func(options *DbOptions) { - options.client = c - } -} diff --git a/db/mongo/db_test.go b/db/mongo/db_test.go deleted file mode 100644 index a2bbbbc3..00000000 --- a/db/mongo/db_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package mongo - -import ( - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "testing" -) - -func TestMongoGetDb(t *testing.T) { - dbName := "test_db" - viper.Set("mongo.db", dbName) - err := InitMongo() - require.Nil(t, err) - - db := GetMongoDb("") - require.Equal(t, dbName, db.Name()) -} diff --git a/docker/base-image/install/go/go.sh b/docker/base-image/install/go/go.sh index 07087d19..63c1e73b 100644 --- a/docker/base-image/install/go/go.sh +++ b/docker/base-image/install/go/go.sh @@ -19,6 +19,10 @@ eval "$(goenv init -)" goenv install ${version} goenv global ${version} +# Create symbolic links +ln -sf "$(goenv which go)" /usr/local/bin/go +ln -sf "$(goenv which gofmt)" /usr/local/bin/gofmt + # verify go_version=$(go version) if [[ $go_version =~ "go${version}" ]]; then diff --git a/docker/base-image/install/java/java.sh b/docker/base-image/install/java/java.sh index e2bb04c8..69f51353 100644 --- a/docker/base-image/install/java/java.sh +++ b/docker/base-image/install/java/java.sh @@ -1,10 +1,35 @@ #!/bin/bash -# Exit on error -set -e - -# Update package list and install OpenJDK 11 -DEBIAN_FRONTEND=noninteractive apt-get update && \ - apt-get install -y --no-install-recommends openjdk-11-jdk && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +version="11.0.12-open" + +# Install SDKMAN! +curl -s "https://get.sdkman.io" | bash + +# Source SDKMAN! +source "$HOME/.sdkman/bin/sdkman-init.sh" + +# Install Java 11 (you can specify vendor, e.g., 11.0.12-open for OpenJDK) +sdk install java ${version} + +# Set Java 11 as default +sdk default java ${version} + +# Create symbolic links +ln -sf "$(sdkman which java)" /usr/local/bin/java +ln -sf "$(sdkman which javac)" /usr/local/bin/javac + +# Verify +java_version=$(java -version) +if [[ $java_version =~ "${version}" ]]; then + : +else + echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\"" + exit 1 +fi +javac_version=$(javac -version) +if [[ $javac_version =~ "${version}" ]]; then + : +else + echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\"" + exit 1 +fi \ No newline at end of file diff --git a/docker/base-image/install/node/node.sh b/docker/base-image/install/node/node.sh index e8caa419..65c65637 100644 --- a/docker/base-image/install/node/node.sh +++ b/docker/base-image/install/node/node.sh @@ -22,6 +22,12 @@ nvm install ${version} nvm use ${version} nvm alias default ${version} +# Create symbolic links +ln -sf "$(nvm which node)" /usr/local/bin/node +ln -sf "$(nvm which npm)" /usr/local/bin/npm +ln -sf "$(nvm which yarn)" /usr/local/bin/yarn +ln -sf "$(nvm which pnpm)" /usr/local/bin/pnpm + # verifies the right Node.js version is in the environment if [[ ! "$(node -v)" =~ ^v${version} ]]; then echo "Node.js version is not v${version}.x" diff --git a/docker/base-image/install/python/python.sh b/docker/base-image/install/python/python.sh index 5ce3aac8..eeef7dc4 100644 --- a/docker/base-image/install/python/python.sh +++ b/docker/base-image/install/python/python.sh @@ -41,6 +41,10 @@ eval "$(pyenv virtualenv-init -)" pyenv install ${version} pyenv global ${version} +# Create symbolic links +ln -sf $(pyenv which python) /usr/local/bin/python +ln -sf $(pyenv which pip) /usr/local/bin/pip + # verify python_version=$(python -V) if [[ $python_version =~ "Python ${version}" ]]; then