diff --git a/constants/data_field.go b/constants/data_field.go new file mode 100644 index 0000000..604ad1d --- /dev/null +++ b/constants/data_field.go @@ -0,0 +1,12 @@ +package constants + +const ( + DataFieldTypeGeneral = "general" + DataFieldTypeNumeric = "numeric" + DataFieldTypeDate = "date" + DataFieldTypeCurrency = "currency" + DataFieldTypeUrl = "url" + DataFieldTypeImage = "image" + DataFieldTypeAudio = "audio" + DataFieldTypeVideo = "video" +) diff --git a/controllers/binder_json.go b/controllers/binder_json.go index cc3ae4f..e00ec33 100644 --- a/controllers/binder_json.go +++ b/controllers/binder_json.go @@ -60,6 +60,9 @@ func (b *JsonBinder) Bind(c *gin.Context) (res interfaces.Model, err error) { case ControllerIdDataSource: err = c.ShouldBindJSON(&m.DataSource) return &m.DataSource, nil + case ControllerIdDataCollection: + err = c.ShouldBindJSON(&m.DataCollection) + return &m.DataCollection, nil case ControllerIdPlugin: err = c.ShouldBindJSON(&m.Plugin) return &m.Plugin, nil @@ -119,6 +122,9 @@ func (b *JsonBinder) BindList(c *gin.Context) (res interface{}, err error) { case ControllerIdDataSource: err = c.ShouldBindJSON(&m.DataSources) return m.DataSources, nil + case ControllerIdDataCollection: + err = c.ShouldBindJSON(&m.DataCollections) + return m.DataCollections, nil case ControllerIdPlugin: err = c.ShouldBindJSON(&m.Plugins) return m.Plugins, nil @@ -190,6 +196,9 @@ func (b *JsonBinder) BindBatchRequestPayloadWithStringData(c *gin.Context) (payl case ControllerIdDataSource: err = json.Unmarshal([]byte(payload.Data), &m.DataSource) return payload, &m.DataSource, err + case ControllerIdDataCollection: + err = json.Unmarshal([]byte(payload.Data), &m.DataCollection) + return payload, &m.DataCollection, err case ControllerIdPlugin: err = json.Unmarshal([]byte(payload.Data), &m.Plugin) return payload, &m.Plugin, err diff --git a/controllers/data_collection.go b/controllers/data_collection.go index a043e15..8bdd034 100644 --- a/controllers/data_collection.go +++ b/controllers/data_collection.go @@ -1,3 +1,71 @@ package controllers -var DataCollectionController ListController +import ( + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/service" + "go.uber.org/dig" +) + +var DataCollectionController *dataCollectionController + +func getDataCollectionActions() []Action { + //ctx := newDataCollectionContext() + return []Action{} +} + +type dataCollectionController struct { + ListActionControllerDelegate + d ListActionControllerDelegate + ctx *dataCollectionContext +} + +type dataCollectionContext struct { + modelSvc service.ModelService + resultSvc interfaces.ResultService +} + +var _dataCollectionCtx *dataCollectionContext + +func newDataCollectionContext() *dataCollectionContext { + if _dataCollectionCtx != nil { + return _dataCollectionCtx + } + + // context + ctx := &dataCollectionContext{} + + // dependency injection + c := dig.New() + if err := c.Provide(service.NewService); err != nil { + panic(err) + } + if err := c.Invoke(func( + modelSvc service.ModelService, + ) { + ctx.modelSvc = modelSvc + }); err != nil { + panic(err) + } + + _dataCollectionCtx = ctx + + return ctx +} + +func newDataCollectionController() *dataCollectionController { + actions := getDataCollectionActions() + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions) + d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions) + ctx := newDataCollectionContext() + + return &dataCollectionController{ + ListActionControllerDelegate: *ctr, + d: *d, + ctx: ctx, + } +} diff --git a/controllers/init.go b/controllers/init.go index 73b0ce8..e55385a 100644 --- a/controllers/init.go +++ b/controllers/init.go @@ -21,7 +21,7 @@ func InitControllers() (err error) { LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions()) ColorController = NewActionControllerDelegate(ControllerIdColor, getColorActions()) PluginController = newPluginController() - DataCollectionController = NewListControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection)) + DataCollectionController = newDataCollectionController() ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions()) ScheduleController = newScheduleController() StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions()) diff --git a/controllers/result.go b/controllers/result.go index 4c15576..c8cd997 100644 --- a/controllers/result.go +++ b/controllers/result.go @@ -4,6 +4,7 @@ import ( "github.com/crawlab-team/crawlab-core/models/models" "github.com/crawlab-team/crawlab-core/models/service" "github.com/crawlab-team/crawlab-core/result" + "github.com/crawlab-team/crawlab-core/utils" "github.com/crawlab-team/crawlab-db/generic" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" @@ -86,7 +87,7 @@ func (ctx *resultContext) getList(c *gin.Context) { // params pagination := MustGetPagination(c) - query := generic.ListQuery{} // TODO: implement query + query := ctx.getListQuery(c) // get results data, err := svc.List(query, &generic.ListOptions{ @@ -120,6 +121,21 @@ func (ctx *resultContext) getList(c *gin.Context) { HandleSuccessWithListData(c, data, total) } +func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) { + f, err := GetFilter(c) + if err != nil { + return q + } + for _, cond := range f.Conditions { + q = append(q, generic.ListQueryCondition{ + Key: cond.Key, + Op: cond.Op, + Value: utils.NormalizeObjectId(cond.Value), + }) + } + return q +} + func newResultContext() *resultContext { // context ctx := &resultContext{} diff --git a/controllers/task.go b/controllers/task.go index 63da202..1c1c0f5 100644 --- a/controllers/task.go +++ b/controllers/task.go @@ -2,6 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab-core/config" + "github.com/crawlab-team/crawlab-core/constants" "github.com/crawlab-team/crawlab-core/entity" "github.com/crawlab-team/crawlab-core/errors" "github.com/crawlab-team/crawlab-core/interfaces" @@ -63,6 +64,10 @@ type taskController struct { ctx *taskContext } +func (ctr *taskController) Get(c *gin.Context) { + ctr.ctx.getWithStatsSpider(c) +} + func (ctr *taskController) GetList(c *gin.Context) { withStats := c.Query("stats") if withStats == "" { @@ -302,6 +307,40 @@ func (ctx *taskContext) getListWithStats(c *gin.Context) { HandleSuccessWithListData(c, data, total) } +func (ctx *taskContext) getWithStatsSpider(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := ctx.modelSvc.GetTaskById(id) + if err == mongo2.ErrNoDocuments { + HandleErrorNotFound(c, err) + return + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // spider + t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId) + + // skip if task status is pending + if t.Status == constants.TaskStatusPending { + HandleSuccessWithData(c, t) + return + } + + // task stat + t.Stat, _ = ctx.modelSvc.GetTaskStatById(id) + + HandleSuccessWithData(c, t) +} + func (ctx *taskContext) getData(c *gin.Context) { // id id, err := primitive.ObjectIDFromHex(c.Param("id")) diff --git a/entity/data_field.go b/entity/data_field.go new file mode 100644 index 0000000..017aee2 --- /dev/null +++ b/entity/data_field.go @@ -0,0 +1,6 @@ +package entity + +type DataField struct { + Key string `json:"key" bson:"key"` + Type string `json:"type" bson:"type"` +} diff --git a/go.mod b/go.mod index 7a3b148..2e2680d 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/matcornic/hermes/v2 v2.1.0 github.com/mitchellh/go-homedir v1.1.0 + github.com/nleeper/goment v1.4.4 // indirect github.com/olivere/elastic/v7 v7.0.15 github.com/pelletier/go-toml/v2 v2.0.2 // indirect github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index a886886..e344b97 100644 --- a/go.sum +++ b/go.sum @@ -493,6 +493,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nleeper/goment v1.4.4 h1:GlMTpxvhueljArSunzYjN9Ri4SOmpn0Vh2hg2z/IIl8= +github.com/nleeper/goment v1.4.4/go.mod h1:zDl5bAyDhqxwQKAvkSXMRLOdCowrdZz53ofRJc4VhTo= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -628,6 +630,8 @@ github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tkuchiki/go-timezone v0.2.0 h1:yyZVHtQRVZ+wvlte5HXvSpBkR0dPYnPEIgq9qqAqltk= +github.com/tkuchiki/go-timezone v0.2.0/go.mod h1:b1Ean9v2UXtxSq4TZF0i/TU9NuoWa9hOzOKoGCV2zqY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= diff --git a/models/models/data_collection.go b/models/models/data_collection.go index 190bcd1..3a03d7d 100644 --- a/models/models/data_collection.go +++ b/models/models/data_collection.go @@ -1,13 +1,15 @@ package models import ( + "github.com/crawlab-team/crawlab-core/entity" "github.com/crawlab-team/crawlab-core/interfaces" "go.mongodb.org/mongo-driver/bson/primitive" ) type DataCollection struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Name string `json:"name" bson:"name"` + Id primitive.ObjectID `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Fields []entity.DataField `json:"fields" bson:"fields"` } func (dc *DataCollection) GetId() (id primitive.ObjectID) { diff --git a/models/models/data_field.go b/models/models/data_field.go deleted file mode 100644 index b5d50b9..0000000 --- a/models/models/data_field.go +++ /dev/null @@ -1,9 +0,0 @@ -package models - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type DataField struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` -} diff --git a/models/models/task.go b/models/models/task.go index 0cd4a3a..ced11bc 100644 --- a/models/models/task.go +++ b/models/models/task.go @@ -23,6 +23,7 @@ type Task struct { Stat *TaskStat `json:"stat,omitempty" bson:"-"` HasSub bool `json:"has_sub" json:"has_sub"` // whether to have sub-tasks SubTasks []Task `json:"sub_tasks,omitempty" bson:"-"` + Spider *Spider `json:"spider,omitempty" bson:"-"` UserId primitive.ObjectID `json:"-" bson:"-"` } diff --git a/utils/bson.go b/utils/bson.go index 24a20aa..e6443e8 100644 --- a/utils/bson.go +++ b/utils/bson.go @@ -110,3 +110,16 @@ func DenormalizeBsonMObjectId(m bson.M) (res bson.M) { } return m } + +func NormalizeObjectId(v interface{}) (res interface{}) { + switch v.(type) { + case string: + oid, err := primitive.ObjectIDFromHex(v.(string)) + if err != nil { + return v + } + return oid + default: + return v + } +}