feat: added modules

core/controllers/test/base.go

@@ -0,0 +1,104 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/routes"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gavv/httpexpect/v2"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/dig"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func init() {
+	var err error
+	T, err = NewTest()
+	if err != nil {
+		panic(err)
+	}
+}
+
+type Test struct {
+	// dependencies
+	modelSvc service.ModelService
+
+	// internals
+	app *gin.Engine
+	svr *httptest.Server
+
+	// test data
+	TestUsername string
+	TestPassword string
+	TestToken    string
+}
+
+func (t *Test) Setup(t2 *testing.T) {
+	//if err := controllers.InitControllers(); err != nil {
+	//	panic(err)
+	//}
+	//t2.Cleanup(t.Cleanup)
+}
+
+func (t *Test) Cleanup() {
+	_ = t.modelSvc.DropAll()
+	time.Sleep(200 * time.Millisecond)
+}
+
+func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) {
+	e = httpexpect.New(t2, t.svr.URL)
+	res := e.POST("/login").WithJSON(map[string]string{
+		"username": t.TestUsername,
+		"password": t.TestPassword,
+	}).Expect().JSON().Object()
+	t.TestToken = res.Path("$.data").String().Raw()
+	require.NotEmpty(t2, t.TestToken)
+	return e
+}
+
+func (t *Test) WithAuth(req *httpexpect.Request) *httpexpect.Request {
+	return req.WithHeader("Authorization", t.TestToken)
+}
+
+var T *Test
+
+func NewTest() (res *Test, err error) {
+	// test
+	t := &Test{}
+
+	// gin app
+	t.app = gin.New()
+
+	// http test server
+	t.svr = httptest.NewServer(t.app)
+
+	// init controllers
+	if err := controllers.InitControllers(); err != nil {
+		return nil, err
+	}
+
+	// init routes
+	if err := routes.InitRoutes(t.app); err != nil {
+		return nil, err
+	}
+
+	// dependency injection
+	c := dig.New()
+	if err := c.Provide(service.NewService); err != nil {
+		return nil, trace.TraceError(err)
+	}
+	if err := c.Invoke(func(modelSvc service.ModelService) {
+		t.modelSvc = modelSvc
+	}); err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// test data
+	t.TestUsername = "admin"
+	t.TestPassword = "admin"
+
+	return t, nil
+}

core/controllers/test/export_test.go

@@ -0,0 +1,59 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func init() {
+	viper.Set("mongo.db", "crawlab_test")
+}
+
+func TestExportController_Csv(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	// mongo collection
+	colName := "test_collection_for_export"
+	col := mongo.GetMongoCol(colName)
+
+	// insert test data to mongo collection
+	for i := 0; i < 10; i++ {
+		_, err := col.Insert(bson.M{
+			"field1": i + 1,
+			"field2": i + 2,
+			"field3": i + 3,
+			"field4": i + 4,
+		})
+		require.Nil(t, err)
+	}
+
+	// export from mongo collection
+	res := T.WithAuth(e.POST("/export/csv")).
+		WithQuery("target", colName).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+
+	// export id
+	exportId := res.Path("$.data").String().Raw()
+
+	// poll export with export id
+	for i := 0; i < 10; i++ {
+		res = T.WithAuth(e.GET("/export/csv/" + exportId)).Expect().Status(http.StatusOK).JSON().Object()
+		status := res.Path("$.data.status").String().Raw()
+		if status == constants.TaskStatusFinished {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	// download exported csv file
+	csvFileBody := T.WithAuth(e.GET("/export/csv/" + exportId + "/download")).Expect().Status(http.StatusOK).Body()
+	csvFileBody.NotEmpty()
+}

core/controllers/test/filter_test.go

@@ -0,0 +1,76 @@
+package test
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"testing"
+)
+
+func init() {
+	viper.Set("mongo.db", "crawlab_test")
+}
+
+func TestFilterController_GetColFieldOptions(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	// mongo collection
+	colName := "test_collection_for_filter"
+	field1 := "field1"
+	field2 := "field2"
+	value1 := "value1"
+	col := mongo.GetMongoCol(colName)
+	n := 10
+	var ids []primitive.ObjectID
+	var names []string
+	for i := 0; i < n; i++ {
+		_id := primitive.NewObjectID()
+		ids = append(ids, _id)
+		name := fmt.Sprintf("name_%d", i)
+		names = append(names, name)
+		_, err := col.Insert(bson.M{field1: value1, field2: i % 2, "name": name, "_id": _id})
+		require.Nil(t, err)
+	}
+
+	// validate filter options field 1
+	res := T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field1, field1))).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+	res.Path("$.data").Array().Length().Equal(1)
+	res.Path("$.data").Array().Element(0).Path("$.value").Equal(value1)
+
+	// validate filter options field 2
+	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+	res.Path("$.data").Array().Length().Equal(2)
+
+	// validate filter options with query
+	conditions := []entity.Condition{{field2, constants.FilterOpEqual, 0}}
+	conditionsJson, err := json.Marshal(conditions)
+	conditionsJsonStr := string(conditionsJson)
+	require.Nil(t, err)
+	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))).
+		WithQuery(constants.FilterQueryFieldConditions, conditionsJsonStr).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+	res.Path("$.data").Array().Length().Equal(1)
+
+	// validate filter options (basic path)
+	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s", colName))).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+	res.Path("$.data").Array().Length().Equal(n)
+	for i := 0; i < n; i++ {
+		res.Path("$.data").Array().Element(i).Object().Value("value").Equal(ids[i])
+		res.Path("$.data").Array().Element(i).Object().Value("label").Equal(names[i])
+	}
+}

core/controllers/test/main_test.go

@@ -0,0 +1,44 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/user"
+	"go.mongodb.org/mongo-driver/mongo"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	// init user
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+	_, err = modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil)
+	if err != nil {
+		if err.Error() != mongo.ErrNoDocuments.Error() {
+			panic(err)
+		}
+		userSvc, err := user.GetUserService()
+		if err != nil {
+			panic(err)
+		}
+		if err := userSvc.Create(&interfaces.UserCreateOptions{
+			Username: constants.DefaultAdminUsername,
+			Password: constants.DefaultAdminPassword,
+			Role:     constants.RoleAdmin,
+		}); err != nil {
+			panic(err)
+		}
+	}
+
+	if err := controllers.InitControllers(); err != nil {
+		panic(err)
+	}
+
+	m.Run()
+
+	T.Cleanup()
+}

core/controllers/test/project_test.go

@@ -0,0 +1,305 @@
+package test
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func TestProjectController_Get(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	p := models.Project{
+		Name: "test project",
+	}
+	res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id := res.Path("$.data._id").String().Raw()
+	oid, err := primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	res = T.WithAuth(e.GET("/projects/" + id)).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data._id").NotNull()
+	res.Path("$.data.name").Equal("test project")
+}
+
+func TestProjectController_Put(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	p := models.Project{
+		Name:        "old name",
+		Description: "old description",
+	}
+
+	// add
+	res := T.WithAuth(e.POST("/projects")).
+		WithJSON(p).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id := res.Path("$.data._id").String().Raw()
+	oid, err := primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	// change object
+	p.Id = oid
+	p.Name = "new name"
+	p.Description = "new description"
+
+	// update
+	T.WithAuth(e.PUT("/projects/" + id)).
+		WithJSON(p).
+		Expect().Status(http.StatusOK)
+
+	// check
+	res = T.WithAuth(e.GET("/projects/" + id)).Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data._id").Equal(id)
+	res.Path("$.data.name").Equal("new name")
+	res.Path("$.data.description").Equal("new description")
+}
+
+func TestProjectController_Post(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	p := models.Project{
+		Name:        "test project",
+		Description: "this is a test project",
+	}
+
+	res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data._id").NotNull()
+	res.Path("$.data.name").Equal("test project")
+	res.Path("$.data.description").Equal("this is a test project")
+}
+
+func TestProjectController_Delete(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	p := models.Project{
+		Name:        "test project",
+		Description: "this is a test project",
+	}
+
+	// add
+	res := T.WithAuth(e.POST("/projects")).
+		WithJSON(p).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id := res.Path("$.data._id").String().Raw()
+	oid, err := primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	// get
+	res = T.WithAuth(e.GET("/projects/" + id)).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id = res.Path("$.data._id").String().Raw()
+	oid, err = primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	// delete
+	T.WithAuth(e.DELETE("/projects/" + id)).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+
+	// get
+	T.WithAuth(e.GET("/projects/" + id)).
+		Expect().Status(http.StatusNotFound)
+}
+
+func TestProjectController_GetList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	n := 100 // total
+	bn := 10 // batch
+
+	for i := 0; i < n; i++ {
+		p := models.Project{
+			Name: fmt.Sprintf("test name %d", i+1),
+		}
+		obj := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
+		obj.Path("$.data._id").NotNull()
+	}
+
+	f := entity.Filter{
+		//IsOr: false,
+		Conditions: []*entity.Condition{
+			{Key: "name", Op: constants.FilterOpContains, Value: "test name"},
+		},
+	}
+	condBytes, err := json.Marshal(&f.Conditions)
+	require.Nil(t, err)
+
+	pagination := entity.Pagination{
+		Page: 1,
+		Size: bn,
+	}
+
+	// get list with pagination
+	res := T.WithAuth(e.GET("/projects")).
+		WithQuery("conditions", string(condBytes)).
+		WithQueryObject(pagination).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").Array().Length().Equal(bn)
+	res.Path("$.total").Number().Equal(n)
+
+	data := res.Path("$.data").Array()
+	for i := 0; i < bn; i++ {
+		obj := data.Element(i)
+		obj.Path("$.name").Equal(fmt.Sprintf("test name %d", i+1))
+	}
+
+}
+
+func TestProjectController_PostList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	n := 10
+	var docs []models.Project
+	for i := 0; i < n; i++ {
+		docs = append(docs, models.Project{
+			Name:        fmt.Sprintf("project %d", i+1),
+			Description: "this is a project",
+		})
+	}
+
+	T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK)
+
+	res := T.WithAuth(e.GET("/projects")).
+		WithQueryObject(entity.Pagination{Page: 1, Size: 10}).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data").Array().Length().Equal(n)
+}
+
+func TestProjectController_DeleteList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	n := 10
+	var docs []models.Project
+	for i := 0; i < n; i++ {
+		docs = append(docs, models.Project{
+			Name:        fmt.Sprintf("project %d", i+1),
+			Description: "this is a project",
+		})
+	}
+
+	// add
+	res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK).
+		JSON().Object()
+	var ids []primitive.ObjectID
+	data := res.Path("$.data").Array()
+	for i := 0; i < n; i++ {
+		obj := data.Element(i)
+		id := obj.Path("$._id").String().Raw()
+		oid, err := primitive.ObjectIDFromHex(id)
+		require.Nil(t, err)
+		require.False(t, oid.IsZero())
+		ids = append(ids, oid)
+	}
+
+	// delete
+	payload := entity.BatchRequestPayload{
+		Ids: ids,
+	}
+	T.WithAuth(e.DELETE("/projects")).
+		WithJSON(payload).
+		Expect().Status(http.StatusOK)
+
+	// check
+	for _, id := range ids {
+		T.WithAuth(e.GET("/projects/" + id.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+}
+
+func TestProjectController_PutList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	// now
+	now := time.Now()
+
+	n := 10
+	var docs []models.Project
+	for i := 0; i < n; i++ {
+		docs = append(docs, models.Project{
+			Name:        "old name",
+			Description: "old description",
+		})
+	}
+
+	// add
+	res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK).
+		JSON().Object()
+	var ids []primitive.ObjectID
+	data := res.Path("$.data").Array()
+	for i := 0; i < n; i++ {
+		obj := data.Element(i)
+		id := obj.Path("$._id").String().Raw()
+		oid, err := primitive.ObjectIDFromHex(id)
+		require.Nil(t, err)
+		require.False(t, oid.IsZero())
+		ids = append(ids, oid)
+	}
+
+	// wait for 100 millisecond
+	time.Sleep(100 * time.Millisecond)
+
+	// update
+	p := models.Project{
+		Name:        "new name",
+		Description: "new description",
+	}
+	dataBytes, err := json.Marshal(&p)
+	require.Nil(t, err)
+	payload := entity.BatchRequestPayloadWithStringData{
+		Ids:  ids,
+		Data: string(dataBytes),
+		Fields: []string{
+			"name",
+			"description",
+		},
+	}
+	T.WithAuth(e.PUT("/projects")).WithJSON(payload).Expect().Status(http.StatusOK)
+
+	// check response data
+	for i := 0; i < n; i++ {
+		res = T.WithAuth(e.GET("/projects/" + ids[i].Hex())).Expect().Status(http.StatusOK).JSON().Object()
+		res.Path("$.data.name").Equal("new name")
+		res.Path("$.data.description").Equal("new description")
+	}
+
+	// check artifacts
+	pl, err := T.modelSvc.GetProjectList(bson.M{"_id": bson.M{"$in": ids}}, nil)
+	require.Nil(t, err)
+	for _, p := range pl {
+		a, err := delegate.NewModelDelegate(&p).GetArtifact()
+		require.Nil(t, err)
+		require.True(t, a.GetSys().GetUpdateTs().After(now))
+		require.True(t, a.GetSys().GetUpdateTs().After(a.GetSys().GetCreateTs()))
+	}
+}

core/controllers/test/spider_test.go

@@ -0,0 +1,183 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"testing"
+)
+
+func TestSpiderController_Delete(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	s := models.Spider{
+		Name:        "test spider",
+		Description: "this is a test spider",
+		ColName:     "test col name",
+	}
+
+	// add spider
+	res := T.WithAuth(e.POST("/spiders")).
+		WithJSON(s).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id := res.Path("$.data._id").String().Raw()
+	oid, err := primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	// add tasks
+	var taskIds []primitive.ObjectID
+	tasks := []models.Task{
+		{
+			Id:       primitive.NewObjectID(),
+			SpiderId: oid,
+		},
+		{
+			Id:       primitive.NewObjectID(),
+			SpiderId: oid,
+		},
+	}
+	for _, task := range tasks {
+		// add task
+		err := delegate.NewModelDelegate(&task).Add()
+		require.Nil(t, err)
+
+		// add task stat
+		err = delegate.NewModelDelegate(&models.TaskStat{
+			Id: task.Id,
+		}).Add()
+		require.Nil(t, err)
+
+		taskIds = append(taskIds, task.Id)
+	}
+
+	// delete
+	T.WithAuth(e.DELETE("/spiders/" + id)).
+		Expect().Status(http.StatusOK)
+
+	// get
+	T.WithAuth(e.GET("/spiders/" + id)).
+		Expect().Status(http.StatusNotFound)
+
+	// get tasks
+	for _, task := range tasks {
+		T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+	// spider stat
+	modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat)
+	spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{
+		"_id": oid,
+	})
+	require.Nil(t, err)
+	require.Zero(t, spiderStatCount)
+
+	// task stats
+	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
+	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
+		"_id": bson.M{
+			"$in": taskIds,
+		},
+	})
+	require.Nil(t, err)
+	require.Zero(t, taskStatCount)
+}
+
+func TestSpiderController_DeleteList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	spiders := []models.Spider{
+		{
+			Id:          primitive.NewObjectID(),
+			Name:        "test spider 1",
+			Description: "this is a test spider 1",
+			ColName:     "test col name 1",
+		},
+		{
+			Id:          primitive.NewObjectID(),
+			Name:        "test spider 2",
+			Description: "this is a test spider 2",
+			ColName:     "test col name 2",
+		},
+	}
+
+	// add spiders
+	for _, spider := range spiders {
+		T.WithAuth(e.POST("/spiders")).
+			WithJSON(spider).
+			Expect().Status(http.StatusOK)
+	}
+
+	var spiderIds []primitive.ObjectID
+	var taskIds []primitive.ObjectID
+	for _, spider := range spiders {
+		// task id
+		taskId := primitive.NewObjectID()
+
+		// add task
+		err := delegate.NewModelDelegate(&models.Task{
+			Id:       taskId,
+			SpiderId: spider.Id,
+		}).Add()
+		require.Nil(t, err)
+
+		// add task stats
+		err = delegate.NewModelDelegate(&models.TaskStat{
+			Id: taskId,
+		}).Add()
+		require.Nil(t, err)
+
+		spiderIds = append(spiderIds, spider.Id)
+		taskIds = append(taskIds, taskId)
+	}
+
+	// delete spiders
+	T.WithAuth(e.DELETE("/spiders")).
+		WithJSON(entity.BatchRequestPayload{
+			Ids: spiderIds,
+		}).Expect().Status(http.StatusOK)
+
+	// get spiders
+	for _, spider := range spiders {
+		// get
+		T.WithAuth(e.GET("/spiders/" + spider.Id.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+	// get tasks
+	for _, taskId := range taskIds {
+		T.WithAuth(e.GET("/tasks/" + taskId.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+	// spider stat
+	modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat)
+	spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{
+		"_id": bson.M{
+			"$in": spiderIds,
+		},
+	})
+	require.Nil(t, err)
+	require.Zero(t, spiderStatCount)
+
+	// task stats
+	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
+	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
+		"_id": bson.M{
+			"$in": taskIds,
+		},
+	})
+	require.Nil(t, err)
+	require.Zero(t, taskStatCount)
+}

core/controllers/test/task_test.go

@@ -0,0 +1,102 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"testing"
+)
+
+func TestTaskController_Delete(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	task := models.Task{
+		Id: primitive.NewObjectID(),
+	}
+
+	// add task
+	err := delegate.NewModelDelegate(&task).Add()
+	require.Nil(t, err)
+
+	// add task stat
+	err = delegate.NewModelDelegate(&models.TaskStat{
+		Id: task.Id,
+	}).Add()
+	require.Nil(t, err)
+
+	// delete
+	T.WithAuth(e.DELETE("/tasks/" + task.Id.Hex())).
+		Expect().Status(http.StatusOK)
+
+	// get
+	T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
+		Expect().Status(http.StatusNotFound)
+
+	// task stats
+	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
+	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
+		"_id": task.Id,
+	})
+	require.Nil(t, err)
+	require.Zero(t, taskStatCount)
+}
+
+func TestTaskController_DeleteList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	tasks := []models.Task{
+		{
+			Id: primitive.NewObjectID(),
+		},
+		{
+			Id: primitive.NewObjectID(),
+		},
+	}
+
+	// add spiders
+	var taskIds []primitive.ObjectID
+	for _, task := range tasks {
+		// add task
+		err := delegate.NewModelDelegate(&task).Add()
+		require.Nil(t, err)
+
+		// add task stat
+		err = delegate.NewModelDelegate(&models.TaskStat{
+			Id: task.Id,
+		}).Add()
+		require.Nil(t, err)
+
+		taskIds = append(taskIds, task.Id)
+	}
+
+	// delete tasks
+	T.WithAuth(e.DELETE("/tasks")).
+		WithJSON(entity.BatchRequestPayload{
+			Ids: taskIds,
+		}).Expect().Status(http.StatusOK)
+
+	// get tasks
+	for _, task := range tasks {
+		// get
+		T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+	// task stats
+	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
+	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
+		"_id": bson.M{
+			"$in": taskIds,
+		},
+	})
+	require.Nil(t, err)
+	require.Zero(t, taskStatCount)
+}