feat(data): added data results preview in various formats

AB#45

constants/data_field.go

@@ -0,0 +1,12 @@
+package constants
+
+const (
+	DataFieldTypeGeneral  = "general"
+	DataFieldTypeNumeric  = "numeric"
+	DataFieldTypeDate     = "date"
+	DataFieldTypeCurrency = "currency"
+	DataFieldTypeUrl      = "url"
+	DataFieldTypeImage    = "image"
+	DataFieldTypeAudio    = "audio"
+	DataFieldTypeVideo    = "video"
+)

controllers/binder_json.go

@@ -60,6 +60,9 @@ func (b *JsonBinder) Bind(c *gin.Context) (res interfaces.Model, err error) {
 	case ControllerIdDataSource:
 		err = c.ShouldBindJSON(&m.DataSource)
 		return &m.DataSource, nil
+	case ControllerIdDataCollection:
+		err = c.ShouldBindJSON(&m.DataCollection)
+		return &m.DataCollection, nil
 	case ControllerIdPlugin:
 		err = c.ShouldBindJSON(&m.Plugin)
 		return &m.Plugin, nil
@@ -119,6 +122,9 @@ func (b *JsonBinder) BindList(c *gin.Context) (res interface{}, err error) {
 	case ControllerIdDataSource:
 		err = c.ShouldBindJSON(&m.DataSources)
 		return m.DataSources, nil
+	case ControllerIdDataCollection:
+		err = c.ShouldBindJSON(&m.DataCollections)
+		return m.DataCollections, nil
 	case ControllerIdPlugin:
 		err = c.ShouldBindJSON(&m.Plugins)
 		return m.Plugins, nil
@@ -190,6 +196,9 @@ func (b *JsonBinder) BindBatchRequestPayloadWithStringData(c *gin.Context) (payl
 	case ControllerIdDataSource:
 		err = json.Unmarshal([]byte(payload.Data), &m.DataSource)
 		return payload, &m.DataSource, err
+	case ControllerIdDataCollection:
+		err = json.Unmarshal([]byte(payload.Data), &m.DataCollection)
+		return payload, &m.DataCollection, err
 	case ControllerIdPlugin:
 		err = json.Unmarshal([]byte(payload.Data), &m.Plugin)
 		return payload, &m.Plugin, err

controllers/data_collection.go

@@ -1,3 +1,71 @@
 package controllers
 
-var DataCollectionController ListController
+import (
+	"github.com/crawlab-team/crawlab-core/interfaces"
+	"github.com/crawlab-team/crawlab-core/models/service"
+	"go.uber.org/dig"
+)
+
+var DataCollectionController *dataCollectionController
+
+func getDataCollectionActions() []Action {
+	//ctx := newDataCollectionContext()
+	return []Action{}
+}
+
+type dataCollectionController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *dataCollectionContext
+}
+
+type dataCollectionContext struct {
+	modelSvc  service.ModelService
+	resultSvc interfaces.ResultService
+}
+
+var _dataCollectionCtx *dataCollectionContext
+
+func newDataCollectionContext() *dataCollectionContext {
+	if _dataCollectionCtx != nil {
+		return _dataCollectionCtx
+	}
+
+	// context
+	ctx := &dataCollectionContext{}
+
+	// dependency injection
+	c := dig.New()
+	if err := c.Provide(service.NewService); err != nil {
+		panic(err)
+	}
+	if err := c.Invoke(func(
+		modelSvc service.ModelService,
+	) {
+		ctx.modelSvc = modelSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	_dataCollectionCtx = ctx
+
+	return ctx
+}
+
+func newDataCollectionController() *dataCollectionController {
+	actions := getDataCollectionActions()
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
+	d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
+	ctx := newDataCollectionContext()
+
+	return &dataCollectionController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

controllers/init.go

@@ -21,7 +21,7 @@ func InitControllers() (err error) {
 	LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions())
 	ColorController = NewActionControllerDelegate(ControllerIdColor, getColorActions())
 	PluginController = newPluginController()
-	DataCollectionController = NewListControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection))
+	DataCollectionController = newDataCollectionController()
 	ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions())
 	ScheduleController = newScheduleController()
 	StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions())

controllers/result.go

@@ -4,6 +4,7 @@ import (
 	"github.com/crawlab-team/crawlab-core/models/models"
 	"github.com/crawlab-team/crawlab-core/models/service"
 	"github.com/crawlab-team/crawlab-core/result"
+	"github.com/crawlab-team/crawlab-core/utils"
 	"github.com/crawlab-team/crawlab-db/generic"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
@@ -86,7 +87,7 @@ func (ctx *resultContext) getList(c *gin.Context) {
 
 	// params
 	pagination := MustGetPagination(c)
-	query := generic.ListQuery{} // TODO: implement query
+	query := ctx.getListQuery(c)
 
 	// get results
 	data, err := svc.List(query, &generic.ListOptions{
@@ -120,6 +121,21 @@ func (ctx *resultContext) getList(c *gin.Context) {
 	HandleSuccessWithListData(c, data, total)
 }
 
+func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) {
+	f, err := GetFilter(c)
+	if err != nil {
+		return q
+	}
+	for _, cond := range f.Conditions {
+		q = append(q, generic.ListQueryCondition{
+			Key:   cond.Key,
+			Op:    cond.Op,
+			Value: utils.NormalizeObjectId(cond.Value),
+		})
+	}
+	return q
+}
+
 func newResultContext() *resultContext {
 	// context
 	ctx := &resultContext{}

controllers/task.go

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"github.com/crawlab-team/crawlab-core/config"
+	"github.com/crawlab-team/crawlab-core/constants"
 	"github.com/crawlab-team/crawlab-core/entity"
 	"github.com/crawlab-team/crawlab-core/errors"
 	"github.com/crawlab-team/crawlab-core/interfaces"
@@ -63,6 +64,10 @@ type taskController struct {
 	ctx *taskContext
 }
 
+func (ctr *taskController) Get(c *gin.Context) {
+	ctr.ctx.getWithStatsSpider(c)
+}
+
 func (ctr *taskController) GetList(c *gin.Context) {
 	withStats := c.Query("stats")
 	if withStats == "" {
@@ -302,6 +307,40 @@ func (ctx *taskContext) getListWithStats(c *gin.Context) {
 	HandleSuccessWithListData(c, data, total)
 }
 
+func (ctx *taskContext) getWithStatsSpider(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := ctx.modelSvc.GetTaskById(id)
+	if err == mongo2.ErrNoDocuments {
+		HandleErrorNotFound(c, err)
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// spider
+	t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId)
+
+	// skip if task status is pending
+	if t.Status == constants.TaskStatusPending {
+		HandleSuccessWithData(c, t)
+		return
+	}
+
+	// task stat
+	t.Stat, _ = ctx.modelSvc.GetTaskStatById(id)
+
+	HandleSuccessWithData(c, t)
+}
+
 func (ctx *taskContext) getData(c *gin.Context) {
 	// id
 	id, err := primitive.ObjectIDFromHex(c.Param("id"))

entity/data_field.go

@@ -0,0 +1,6 @@
+package entity
+
+type DataField struct {
+	Key  string `json:"key" bson:"key"`
+	Type string `json:"type" bson:"type"`
+}

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matcornic/hermes/v2 v2.1.0
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/nleeper/goment v1.4.4 // indirect
 	github.com/olivere/elastic/v7 v7.0.15
 	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
 	github.com/pkg/errors v0.9.1

go.sum

@@ -493,6 +493,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nleeper/goment v1.4.4 h1:GlMTpxvhueljArSunzYjN9Ri4SOmpn0Vh2hg2z/IIl8=
+github.com/nleeper/goment v1.4.4/go.mod h1:zDl5bAyDhqxwQKAvkSXMRLOdCowrdZz53ofRJc4VhTo=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
@@ -628,6 +630,8 @@ github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev
 github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
 github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
 github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
+github.com/tkuchiki/go-timezone v0.2.0 h1:yyZVHtQRVZ+wvlte5HXvSpBkR0dPYnPEIgq9qqAqltk=
+github.com/tkuchiki/go-timezone v0.2.0/go.mod h1:b1Ean9v2UXtxSq4TZF0i/TU9NuoWa9hOzOKoGCV2zqY=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo=

models/models/data_collection.go

@@ -1,13 +1,15 @@
 package models
 
 import (
+	"github.com/crawlab-team/crawlab-core/entity"
 	"github.com/crawlab-team/crawlab-core/interfaces"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
 type DataCollection struct {
-	Id   primitive.ObjectID `json:"_id" bson:"_id"`
-	Name string             `json:"name" bson:"name"`
+	Id     primitive.ObjectID `json:"_id" bson:"_id"`
+	Name   string             `json:"name" bson:"name"`
+	Fields []entity.DataField `json:"fields" bson:"fields"`
 }
 
 func (dc *DataCollection) GetId() (id primitive.ObjectID) {

models/models/task.go

@@ -23,6 +23,7 @@ type Task struct {
 	Stat       *TaskStat            `json:"stat,omitempty" bson:"-"`
 	HasSub     bool                 `json:"has_sub" json:"has_sub"` // whether to have sub-tasks
 	SubTasks   []Task               `json:"sub_tasks,omitempty" bson:"-"`
+	Spider     *Spider              `json:"spider,omitempty" bson:"-"`
 	UserId     primitive.ObjectID   `json:"-" bson:"-"`
 }
 

utils/bson.go

@@ -110,3 +110,16 @@ func DenormalizeBsonMObjectId(m bson.M) (res bson.M) {
 	}
 	return m
 }
+
+func NormalizeObjectId(v interface{}) (res interface{}) {
+	switch v.(type) {
+	case string:
+		oid, err := primitive.ObjectIDFromHex(v.(string))
+		if err != nil {
+			return v
+		}
+		return oid
+	default:
+		return v
+	}
+}