git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Remove unused code and update models and functions for notification channels and settings

Browse Source
This commit is contained in:
Marvin Zhang
2024-07-23 12:22:59 +08:00
parent c893b5452e
commit 9ffdd3f1cd
14 changed files with 169 additions and 832 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
core/task/base.go
View File

@@ -1,97 +0,0 @@
package task
import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
)
type BaseService struct {
// dependencies
interfaces.WithConfigPath
modelSvc service.ModelService
// internals
stopped bool
}
func (svc *BaseService) Init() error {
// implement me
return nil
}
func (svc *BaseService) Start() {
// implement me
}
func (svc *BaseService) Wait() {
utils.DefaultWait()
}
func (svc *BaseService) Stop() {
svc.stopped = true
}
// SaveTask deprecated
func (svc *BaseService) SaveTask(t interfaces.Task, status string) (err error) {
// normalize status
if status == "" {
status = constants.TaskStatusPending
}
// set task status
t.SetStatus(status)
// attempt to get task from database
_, err = svc.modelSvc.GetTaskById(t.GetId())
if err != nil {
// if task does not exist, add to database
if err == mongo.ErrNoDocuments {
if err := delegate.NewModelDelegate(t).Add(); err != nil {
return err
}
return nil
} else {
return err
}
} else {
// otherwise, update
if err := delegate.NewModelDelegate(t).Save(); err != nil {
return err
}
return nil
}
}
func (svc *BaseService) IsStopped() (res bool) {
return svc.stopped
}
func (svc *BaseService) GetQueue(nodeId primitive.ObjectID) (queue string) {
if nodeId.IsZero() {
return fmt.Sprintf("%s", constants.TaskListQueuePrefixPublic)
} else {
return fmt.Sprintf("%s:%s", constants.TaskListQueuePrefixNodes, nodeId.Hex())
}
}
func NewBaseService() (svc2 interfaces.TaskBaseService, err error) {
svc := &BaseService{}
// dependency injection
if err := container.GetContainer().Invoke(func(cfgPath interfaces.WithConfigPath, modelSvc service.ModelService) {
svc.WithConfigPath = cfgPath
svc.modelSvc = modelSvc
}); err != nil {
return nil, trace.TraceError(err)
}
return svc, nil
}

264
core/task/scheduler/service.go
View File

@@ -1,264 +0,0 @@
package scheduler
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/task"
"github.com/crawlab-team/crawlab/db/mongo"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
"time"
)
type Service struct {
// dependencies
interfaces.TaskBaseService
nodeCfgSvc interfaces.NodeConfigService
modelSvc service.ModelService
svr interfaces.GrpcServer
handlerSvc interfaces.TaskHandlerService
// settings
interval time.Duration
}
func (svc *Service) Start() {
go svc.initTaskStatus()
go svc.cleanupTasks()
svc.Wait()
svc.Stop()
}
func (svc *Service) Enqueue(t interfaces.Task) (t2 interfaces.Task, err error) {
// set task status
t.SetStatus(constants.TaskStatusPending)
// user
var u *models.User
if !t.GetUserId().IsZero() {
u, _ = svc.modelSvc.GetUserById(t.GetUserId())
}
// add task
if err = delegate.NewModelDelegate(t, u).Add(); err != nil {
return nil, err
}
// task queue item
tq := &models.TaskQueueItem{
Id: t.GetId(),
Priority: t.GetPriority(),
NodeId: t.GetNodeId(),
}
// task stat
ts := &models.TaskStat{
Id: t.GetId(),
CreateTs: time.Now(),
}
// enqueue task
_, err = mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).Insert(tq)
if err != nil {
return nil, trace.TraceError(err)
}
// add task stat
_, err = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Insert(ts)
if err != nil {
return nil, trace.TraceError(err)
}
// success
return t, nil
}
func (svc *Service) Cancel(id primitive.ObjectID, args ...interface{}) (err error) {
// task
t, err := svc.modelSvc.GetTaskById(id)
if err != nil {
return trace.TraceError(err)
}
// initial status
initialStatus := t.Status
// set task status as "cancelled"
_ = svc.SaveTask(t, constants.TaskStatusCancelled)
// set status of pending tasks as "cancelled" and remove from task item queue
if initialStatus == constants.TaskStatusPending {
// remove from task item queue
if err := mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).DeleteId(t.GetId()); err != nil {
return trace.TraceError(err)
}
return nil
}
// whether task is running on master node
isMasterTask, err := svc.isMasterNode(t)
if err != nil {
// when error, force status being set as "cancelled"
return svc.SaveTask(t, constants.TaskStatusCancelled)
}
// node
n, err := svc.modelSvc.GetNodeById(t.GetNodeId())
if err != nil {
return trace.TraceError(err)
}
if isMasterTask {
// cancel task on master
if err := svc.handlerSvc.Cancel(id); err != nil {
return trace.TraceError(err)
}
// cancel success
return nil
} else {
// send to cancel task on worker nodes
if err := svc.svr.SendStreamMessageWithData("node:"+n.GetKey(), grpc.StreamMessageCode_CANCEL_TASK, t); err != nil {
return trace.TraceError(err)
}
// cancel success
return nil
}
}
func (svc *Service) SetInterval(interval time.Duration) {
svc.interval = interval
}
// initTaskStatus initialize task status of existing tasks
func (svc *Service) initTaskStatus() {
// set status of running tasks as TaskStatusAbnormal
runningTasks, err := svc.modelSvc.GetTaskList(bson.M{
"status": bson.M{
"$in": []string{
constants.TaskStatusPending,
constants.TaskStatusRunning,
},
},
}, nil)
if err != nil {
if err == mongo2.ErrNoDocuments {
return
}
trace.PrintError(err)
}
for _, t := range runningTasks {
go func(t *models.Task) {
if err := svc.SaveTask(t, constants.TaskStatusAbnormal); err != nil {
trace.PrintError(err)
}
}(&t)
}
if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskQueue).DeleteList(nil); err != nil {
return
}
}
func (svc *Service) isMasterNode(t *models.Task) (ok bool, err error) {
if t.GetNodeId().IsZero() {
return false, trace.TraceError(errors.ErrorTaskNoNodeId)
}
n, err := svc.modelSvc.GetNodeById(t.GetNodeId())
if err != nil {
if err == mongo2.ErrNoDocuments {
return false, trace.TraceError(errors.ErrorTaskNodeNotFound)
}
return false, trace.TraceError(err)
}
return n.IsMaster, nil
}
func (svc *Service) cleanupTasks() {
for {
// task stats over 30 days ago
taskStats, err := svc.modelSvc.GetTaskStatList(bson.M{
"create_ts": bson.M{
"$lt": time.Now().Add(-30 * 24 * time.Hour),
},
}, nil)
if err != nil {
time.Sleep(30 * time.Minute)
continue
}
// task ids
var ids []primitive.ObjectID
for _, ts := range taskStats {
ids = append(ids, ts.Id)
}
if len(ids) > 0 {
// remove tasks
if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTask).DeleteList(bson.M{
"_id": bson.M{"$in": ids},
}); err != nil {
trace.PrintError(err)
}
// remove task stats
if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskStat).DeleteList(bson.M{
"_id": bson.M{"$in": ids},
}); err != nil {
trace.PrintError(err)
}
}
time.Sleep(30 * time.Minute)
}
}
func NewTaskSchedulerService() (svc2 interfaces.TaskSchedulerService, err error) {
// base service
baseSvc, err := task.NewBaseService()
if err != nil {
return nil, trace.TraceError(err)
}
// service
svc := &Service{
TaskBaseService: baseSvc,
interval: 5 * time.Second,
}
// dependency injection
if err := container.GetContainer().Invoke(func(
nodeCfgSvc interfaces.NodeConfigService,
modelSvc service.ModelService,
svr interfaces.GrpcServer,
handlerSvc interfaces.TaskHandlerService,
) {
svc.nodeCfgSvc = nodeCfgSvc
svc.modelSvc = modelSvc
svc.svr = svr
svc.handlerSvc = handlerSvc
}); err != nil {
return nil, err
}
return svc, nil
}
var svc interfaces.TaskSchedulerService
func GetTaskSchedulerService() (svr interfaces.TaskSchedulerService, err error) {
if svc != nil {
return svc, nil
}
svc, err = NewTaskSchedulerService()
if err != nil {
return nil, err
}
return svc, nil
}

155
core/task/stats/service.go
View File

@@ -1,155 +0,0 @@
package stats
import (
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/task"
"github.com/crawlab-team/crawlab/core/task/log"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"sync"
"time"
)
type Service struct {
// dependencies
interfaces.TaskBaseService
nodeCfgSvc interfaces.NodeConfigService
modelSvc service.ModelService
// internals
mu sync.Mutex
resultServices sync.Map
rsTtl time.Duration
logDriver log.Driver
}
func (svc *Service) Init() (err error) {
go svc.cleanup()
return nil
}
func (svc *Service) InsertData(id primitive.ObjectID, records ...interface{}) (err error) {
resultSvc, err := svc.getResultService(id)
if err != nil {
return err
}
if err := resultSvc.Insert(records...); err != nil {
return err
}
go svc.updateTaskStats(id, len(records))
return nil
}
func (svc *Service) InsertLogs(id primitive.ObjectID, logs ...string) (err error) {
return svc.logDriver.WriteLines(id.Hex(), logs)
}
func (svc *Service) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) {
// atomic operation
svc.mu.Lock()
defer svc.mu.Unlock()
// attempt to get from cache
res, _ := svc.resultServices.Load(id.Hex())
if res != nil {
// hit in cache
resultSvc, ok := res.(interfaces.ResultService)
resultSvc.SetTime(time.Now())
if ok {
return resultSvc, nil
}
}
// task
t, err := svc.modelSvc.GetTaskById(id)
if err != nil {
return nil, err
}
// result service
resultSvc, err = result.GetResultService(t.SpiderId)
if err != nil {
return nil, err
}
// store in cache
svc.resultServices.Store(id.Hex(), resultSvc)
return resultSvc, nil
}
func (svc *Service) updateTaskStats(id primitive.ObjectID, resultCount int) {
_ = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).UpdateId(id, bson.M{
"$inc": bson.M{
"result_count": resultCount,
},
})
}
func (svc *Service) cleanup() {
for {
// atomic operation
svc.mu.Lock()
svc.resultServices.Range(func(key, value interface{}) bool {
rs := value.(interfaces.ResultService)
if time.Now().After(rs.GetTime().Add(svc.rsTtl)) {
svc.resultServices.Delete(key)
}
return true
})
svc.mu.Unlock()
time.Sleep(10 * time.Minute)
}
}
func NewTaskStatsService() (svc2 interfaces.TaskStatsService, err error) {
// base service
baseSvc, err := task.NewBaseService()
if err != nil {
return nil, trace.TraceError(err)
}
// service
svc := &Service{
mu: sync.Mutex{},
TaskBaseService: baseSvc,
resultServices: sync.Map{},
}
// dependency injection
if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService, modelSvc service.ModelService) {
svc.nodeCfgSvc = nodeCfgSvc
svc.modelSvc = modelSvc
}); err != nil {
return nil, trace.TraceError(err)
}
// log driver
svc.logDriver, err = log.GetLogDriver(log.DriverTypeFile)
if err != nil {
return nil, err
}
return svc, nil
}
var _service interfaces.TaskStatsService
func GetTaskStatsService() (svr interfaces.TaskStatsService, err error) {
if _service != nil {
return _service, nil
}
_service, err = NewTaskStatsService()
if err != nil {
return nil, err
}
return _service, nil
}