git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: removed unnecessary code

Browse Source
This commit is contained in:
Marvin Zhang
2024-07-11 12:45:29 +08:00
parent f2f298d8f8
commit aca0c0ebce
57 changed files with 423 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
core/task/handler/runner_v2.go
View File

@@ -15,6 +15,7 @@ import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
service2 "github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/sys_exec"
"github.com/crawlab-team/crawlab/core/utils"
@@ -47,8 +48,8 @@ type RunnerV2 struct {
cmd *exec.Cmd // process command instance
pid int // process id
tid primitive.ObjectID // task id
t *models.TaskV2 // task model.Task
s *models.SpiderV2 // spider model.Spider
t *models2.TaskV2 // task model.Task
s *models2.SpiderV2 // spider model.Spider
ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2
err error // standard process error
envs []models.Env // environment variables
@@ -315,7 +316,7 @@ func (r *RunnerV2) configureEnv() {
}
// global environment variables
envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil)
envs, err := client.NewModelServiceV2[models2.EnvironmentV2]().GetMany(nil, nil)
if err != nil {
trace.PrintError(err)
return
@@ -510,12 +511,12 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) {
r.t.Error = e.Error()
}
if r.svc.GetNodeConfigService().IsMaster() {
err = service2.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t)
err = service2.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t)
if err != nil {
return err
}
} else {
err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t)
err = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t)
if err != nil {
return err
}
@@ -568,7 +569,7 @@ func (r *RunnerV2) writeLogLines(lines []string) {
}
func (r *RunnerV2) _updateTaskStat(status string) {
ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid)
ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid)
if err != nil {
trace.PrintError(err)
return
@@ -589,13 +590,13 @@ func (r *RunnerV2) _updateTaskStat(status string) {
ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds()
}
if r.svc.GetNodeConfigService().IsMaster() {
err = service2.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts)
err = service2.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts)
if err != nil {
trace.PrintError(err)
return
}
} else {
err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts)
err = client.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts)
if err != nil {
trace.PrintError(err)
return
@@ -622,7 +623,7 @@ func (r *RunnerV2) sendNotification() {
func (r *RunnerV2) _updateSpiderStat(status string) {
// task stat
ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid)
ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid)
if err != nil {
trace.PrintError(err)
return
@@ -660,13 +661,13 @@ func (r *RunnerV2) _updateSpiderStat(status string) {
// perform update
if r.svc.GetNodeConfigService().IsMaster() {
err = service2.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update)
err = service2.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update)
if err != nil {
trace.PrintError(err)
return
}
} else {
err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update)
err = client.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update)
if err != nil {
trace.PrintError(err)
return

30
core/task/handler/service_v2.go
View File

@@ -10,7 +10,7 @@ import (
grpcclient "github.com/crawlab-team/crawlab/core/grpc/client"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/trace"
@@ -121,7 +121,7 @@ func (svc *ServiceV2) Fetch() {
t.Error = err.Error()
t.Status = constants.TaskStatusError
t.SetUpdated(t.CreatedBy)
_ = client.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t)
_ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
continue
}
continue
@@ -202,15 +202,15 @@ func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigServic
return svc.cfgSvc
}
func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) {
func (svc *ServiceV2) GetCurrentNode() (n *models2.NodeV2, err error) {
// node key
nodeKey := svc.cfgSvc.GetNodeKey()
// current node
if svc.cfgSvc.IsMaster() {
n, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
n, err = service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
} else {
n, err = client.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
n, err = client.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
}
if err != nil {
return nil, err
@@ -219,11 +219,11 @@ func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) {
return n, nil
}
func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err error) {
func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err error) {
if svc.cfgSvc.IsMaster() {
t, err = service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id)
} else {
t, err = client.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err = client.NewModelServiceV2[models2.TaskV2]().GetById(id)
}
if err != nil {
return nil, err
@@ -232,11 +232,11 @@ func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err
return t, nil
}
func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models.SpiderV2, err error) {
func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2, err error) {
if svc.cfgSvc.IsMaster() {
s, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err = service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
} else {
s, err = client.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err = client.NewModelServiceV2[models2.SpiderV2]().GetById(id)
}
if err != nil {
return nil, err
@@ -267,13 +267,13 @@ func (svc *ServiceV2) getRunnerCount() (count int) {
"status": constants.TaskStatusRunning,
}
if svc.cfgSvc.IsMaster() {
count, err = service.NewModelServiceV2[models.TaskV2]().Count(query)
count, err = service.NewModelServiceV2[models2.TaskV2]().Count(query)
if err != nil {
trace.PrintError(err)
return
}
} else {
count, err = client.NewModelServiceV2[models.TaskV2]().Count(query)
count, err = client.NewModelServiceV2[models2.TaskV2]().Count(query)
if err != nil {
trace.PrintError(err)
return
@@ -323,9 +323,9 @@ func (svc *ServiceV2) reportStatus() (err error) {
// save node
n.SetUpdated(n.CreatedBy)
if svc.cfgSvc.IsMaster() {
err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n)
err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n)
} else {
err = client.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n)
err = client.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n)
}
if err != nil {
return err

42
core/task/scheduler/service_v2.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/grpc/server"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/task/handler"
@@ -36,40 +36,40 @@ func (svc *ServiceV2) Start() {
utils.DefaultWait()
}
func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *models.TaskV2, err error) {
func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *models2.TaskV2, err error) {
// set task status
t.Status = constants.TaskStatusPending
t.SetCreatedBy(by)
t.SetUpdated(by)
// add task
taskModelSvc := service.NewModelServiceV2[models.TaskV2]()
taskModelSvc := service.NewModelServiceV2[models2.TaskV2]()
_, err = taskModelSvc.InsertOne(*t)
if err != nil {
return nil, err
}
// task queue item
tq := models.TaskQueueItemV2{
tq := models2.TaskQueueItemV2{
Priority: t.Priority,
NodeId: t.NodeId,
}
tq.SetId(t.Id)
// task stat
ts := models.TaskStatV2{
ts := models2.TaskStatV2{
CreateTs: time.Now(),
}
ts.SetId(t.Id)
// enqueue task
_, err = service.NewModelServiceV2[models.TaskQueueItemV2]().InsertOne(tq)
_, err = service.NewModelServiceV2[models2.TaskQueueItemV2]().InsertOne(tq)
if err != nil {
return nil, trace.TraceError(err)
}
// add task stat
_, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(ts)
_, err = service.NewModelServiceV2[models2.TaskStatV2]().InsertOne(ts)
if err != nil {
return nil, trace.TraceError(err)
}
@@ -80,7 +80,7 @@ func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *mode
func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err error) {
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if err != nil {
return trace.TraceError(err)
}
@@ -95,7 +95,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err
// set status of pending tasks as "cancelled" and remove from task item queue
if initialStatus == constants.TaskStatusPending {
// remove from task item queue
if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(t.Id); err != nil {
if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(t.Id); err != nil {
return trace.TraceError(err)
}
return nil
@@ -110,7 +110,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err
}
// node
n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId)
n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId)
if err != nil {
return trace.TraceError(err)
}
@@ -136,22 +136,22 @@ func (svc *ServiceV2) SetInterval(interval time.Duration) {
svc.interval = interval
}
func (svc *ServiceV2) SaveTask(t *models.TaskV2, by primitive.ObjectID) (err error) {
func (svc *ServiceV2) SaveTask(t *models2.TaskV2, by primitive.ObjectID) (err error) {
if t.Id.IsZero() {
t.SetCreated(by)
t.SetUpdated(by)
_, err = service.NewModelServiceV2[models.TaskV2]().InsertOne(*t)
_, err = service.NewModelServiceV2[models2.TaskV2]().InsertOne(*t)
return err
} else {
t.SetUpdated(by)
return service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t)
return service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
}
}
// initTaskStatus initialize task status of existing tasks
func (svc *ServiceV2) initTaskStatus() {
// set status of running tasks as TaskStatusAbnormal
runningTasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{
runningTasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{
"status": bson.M{
"$in": []string{
constants.TaskStatusPending,
@@ -166,23 +166,23 @@ func (svc *ServiceV2) initTaskStatus() {
trace.PrintError(err)
}
for _, t := range runningTasks {
go func(t *models.TaskV2) {
go func(t *models2.TaskV2) {
t.Status = constants.TaskStatusAbnormal
if err := svc.SaveTask(t, primitive.NilObjectID); err != nil {
trace.PrintError(err)
}
}(&t)
}
if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteMany(nil); err != nil {
if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteMany(nil); err != nil {
return
}
}
func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) {
func (svc *ServiceV2) isMasterNode(t *models2.TaskV2) (ok bool, err error) {
if t.NodeId.IsZero() {
return false, trace.TraceError(errors.ErrorTaskNoNodeId)
}
n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId)
n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId)
if err != nil {
if errors2.Is(err, mongo2.ErrNoDocuments) {
return false, trace.TraceError(errors.ErrorTaskNodeNotFound)
@@ -195,7 +195,7 @@ func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) {
func (svc *ServiceV2) cleanupTasks() {
for {
// task stats over 30 days ago
taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{
taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{
"create_ts": bson.M{
"$lt": time.Now().Add(-30 * 24 * time.Hour),
},
@@ -213,14 +213,14 @@ func (svc *ServiceV2) cleanupTasks() {
if len(ids) > 0 {
// remove tasks
if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{
if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{
"_id": bson.M{"$in": ids},
}); err != nil {
trace.PrintError(err)
}
// remove task stats
if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{
if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{
"_id": bson.M{"$in": ids},
}); err != nil {
trace.PrintError(err)

6
core/task/stats/service_v2.go
View File

@@ -2,7 +2,7 @@ package stats
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/result"
@@ -63,7 +63,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa
}
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if err != nil {
return nil, err
}
@@ -81,7 +81,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa
}
func (svc *ServiceV2) updateTaskStats(id primitive.ObjectID, resultCount int) {
err := service.NewModelServiceV2[models.TaskStatV2]().UpdateById(id, bson.M{
err := service.NewModelServiceV2[models2.TaskStatV2]().UpdateById(id, bson.M{
"$inc": bson.M{
"result_count": resultCount,
},