git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: fixed dependency errors

Browse Source
This commit is contained in:
Marvin Zhang
2024-11-01 14:08:30 +08:00
parent 8ffba2944d
commit ca5bbe6c8f
56 changed files with 3753 additions and 3791 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
core/task/handler/runner.go
View File

@@ -306,7 +306,7 @@ func (r *Runner) configureEnv() {
}
// global environment variables
envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil)
envs, err := client.NewModelService[models.EnvironmentV2]().GetMany(nil, nil)
if err != nil {
trace.PrintError(err)
return
@@ -501,12 +501,12 @@ func (r *Runner) updateTask(status string, e error) (err error) {
r.t.Error = e.Error()
}
if r.svc.GetNodeConfigService().IsMaster() {
err = service.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t)
err = service.NewModelService[models.TaskV2]().ReplaceById(r.t.Id, *r.t)
if err != nil {
return err
}
} else {
err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t)
err = client.NewModelService[models.TaskV2]().ReplaceById(r.t.Id, *r.t)
if err != nil {
return err
}
@@ -555,7 +555,7 @@ func (r *Runner) writeLogLines(lines []string) {
}
func (r *Runner) _updateTaskStat(status string) {
ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid)
ts, err := client.NewModelService[models.TaskStatV2]().GetById(r.tid)
if err != nil {
trace.PrintError(err)
return
@@ -576,13 +576,13 @@ func (r *Runner) _updateTaskStat(status string) {
ts.TotalDuration = ts.EndTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds()
}
if r.svc.GetNodeConfigService().IsMaster() {
err = service.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts)
err = service.NewModelService[models.TaskStatV2]().ReplaceById(ts.Id, *ts)
if err != nil {
trace.PrintError(err)
return
}
} else {
err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts)
err = client.NewModelService[models.TaskStatV2]().ReplaceById(ts.Id, *ts)
if err != nil {
trace.PrintError(err)
return
@@ -605,7 +605,7 @@ func (r *Runner) sendNotification() {
func (r *Runner) _updateSpiderStat(status string) {
// task stat
ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid)
ts, err := client.NewModelService[models.TaskStatV2]().GetById(r.tid)
if err != nil {
trace.PrintError(err)
return
@@ -643,13 +643,13 @@ func (r *Runner) _updateSpiderStat(status string) {
// perform update
if r.svc.GetNodeConfigService().IsMaster() {
err = service.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update)
err = service.NewModelService[models.SpiderStatV2]().UpdateById(r.s.Id, update)
if err != nil {
trace.PrintError(err)
return
}
} else {
err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update)
err = client.NewModelService[models.SpiderStatV2]().UpdateById(r.s.Id, update)
if err != nil {
trace.PrintError(err)
return

26
core/task/handler/service.go
View File

@@ -110,7 +110,7 @@ func (svc *Service) fetchAndRunTasks() {
t.Error = err.Error()
t.Status = constants.TaskStatusError
t.SetUpdated(t.CreatedBy)
_ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
_ = client.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t)
continue
}
continue
@@ -154,9 +154,9 @@ func (svc *Service) GetCurrentNode() (n *models2.NodeV2, err error) {
// current node
if svc.cfgSvc.IsMaster() {
n, err = service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
n, err = service.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
} else {
n, err = client.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
n, err = client.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
}
if err != nil {
return nil, err
@@ -167,9 +167,9 @@ func (svc *Service) GetCurrentNode() (n *models2.NodeV2, err error) {
func (svc *Service) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err error) {
if svc.cfgSvc.IsMaster() {
t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id)
t, err = service.NewModelService[models2.TaskV2]().GetById(id)
} else {
t, err = client.NewModelServiceV2[models2.TaskV2]().GetById(id)
t, err = client.NewModelService[models2.TaskV2]().GetById(id)
}
if err != nil {
return nil, err
@@ -181,9 +181,9 @@ func (svc *Service) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err e
func (svc *Service) UpdateTask(t *models2.TaskV2) (err error) {
t.SetUpdated(t.CreatedBy)
if svc.cfgSvc.IsMaster() {
err = service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
err = service.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t)
} else {
err = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
err = client.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t)
}
if err != nil {
return err
@@ -193,9 +193,9 @@ func (svc *Service) UpdateTask(t *models2.TaskV2) (err error) {
func (svc *Service) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2, err error) {
if svc.cfgSvc.IsMaster() {
s, err = service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
s, err = service.NewModelService[models2.SpiderV2]().GetById(id)
} else {
s, err = client.NewModelServiceV2[models2.SpiderV2]().GetById(id)
s, err = client.NewModelService[models2.SpiderV2]().GetById(id)
}
if err != nil {
return nil, err
@@ -217,13 +217,13 @@ func (svc *Service) getRunnerCount() (count int) {
},
}
if svc.cfgSvc.IsMaster() {
count, err = service.NewModelServiceV2[models2.TaskV2]().Count(query)
count, err = service.NewModelService[models2.TaskV2]().Count(query)
if err != nil {
trace.PrintError(err)
return
}
} else {
count, err = client.NewModelServiceV2[models2.TaskV2]().Count(query)
count, err = client.NewModelService[models2.TaskV2]().Count(query)
if err != nil {
trace.PrintError(err)
return
@@ -273,9 +273,9 @@ func (svc *Service) updateNodeStatus() (err error) {
// save node
n.SetUpdated(n.CreatedBy)
if svc.cfgSvc.IsMaster() {
err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n)
err = service.NewModelService[models2.NodeV2]().ReplaceById(n.Id, *n)
} else {
err = client.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n)
err = client.NewModelService[models2.NodeV2]().ReplaceById(n.Id, *n)
}
if err != nil {
return err

20
core/task/scheduler/service.go
View File

@@ -44,7 +44,7 @@ func (svc *Service) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *model
t.SetUpdated(by)
// add task
taskModelSvc := service.NewModelServiceV2[models2.TaskV2]()
taskModelSvc := service.NewModelService[models2.TaskV2]()
t.Id, err = taskModelSvc.InsertOne(*t)
if err != nil {
return nil, err
@@ -57,7 +57,7 @@ func (svc *Service) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *model
ts.SetUpdated(by)
// add task stat
_, err = service.NewModelServiceV2[models2.TaskStatV2]().InsertOne(ts)
_, err = service.NewModelService[models2.TaskStatV2]().InsertOne(ts)
if err != nil {
return nil, trace.TraceError(err)
}
@@ -68,7 +68,7 @@ func (svc *Service) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *model
func (svc *Service) Cancel(id, by primitive.ObjectID, force bool) (err error) {
// task
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
t, err := service.NewModelService[models2.TaskV2]().GetById(id)
if err != nil {
log.Errorf("task not found: %s", id.Hex())
return err
@@ -145,18 +145,18 @@ func (svc *Service) SaveTask(t *models2.TaskV2, by primitive.ObjectID) (err erro
if t.Id.IsZero() {
t.SetCreated(by)
t.SetUpdated(by)
_, err = service.NewModelServiceV2[models2.TaskV2]().InsertOne(*t)
_, err = service.NewModelService[models2.TaskV2]().InsertOne(*t)
return err
} else {
t.SetUpdated(by)
return service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
return service.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t)
}
}
// initTaskStatus initialize task status of existing tasks
func (svc *Service) initTaskStatus() {
// set status of running tasks as TaskStatusAbnormal
runningTasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{
runningTasks, err := service.NewModelService[models2.TaskV2]().GetMany(bson.M{
"status": bson.M{
"$in": []string{
constants.TaskStatusPending,
@@ -186,7 +186,7 @@ func (svc *Service) isMasterNode(t *models2.TaskV2) (ok bool, err error) {
if t.NodeId.IsZero() {
return false, trace.TraceError(errors.ErrorTaskNoNodeId)
}
n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId)
n, err := service.NewModelService[models2.NodeV2]().GetById(t.NodeId)
if err != nil {
if errors2.Is(err, mongo2.ErrNoDocuments) {
return false, trace.TraceError(errors.ErrorTaskNodeNotFound)
@@ -199,7 +199,7 @@ func (svc *Service) isMasterNode(t *models2.TaskV2) (ok bool, err error) {
func (svc *Service) cleanupTasks() {
for {
// task stats over 30 days ago
taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{
taskStats, err := service.NewModelService[models2.TaskStatV2]().GetMany(bson.M{
"created_ts": bson.M{
"$lt": time.Now().Add(-30 * 24 * time.Hour),
},
@@ -217,14 +217,14 @@ func (svc *Service) cleanupTasks() {
if len(ids) > 0 {
// remove tasks
if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{
if err := service.NewModelService[models2.TaskV2]().DeleteMany(bson.M{
"_id": bson.M{"$in": ids},
}); err != nil {
trace.PrintError(err)
}
// remove task stats
if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{
if err := service.NewModelService[models2.TaskStatV2]().DeleteMany(bson.M{
"_id": bson.M{"$in": ids},
}); err != nil {
trace.PrintError(err)

6
core/task/stats/service.go
View File

@@ -98,13 +98,13 @@ func (svc *Service) getDatabaseServiceItem(taskId primitive.ObjectID) (item *dat
}
// task
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(taskId)
t, err := service.NewModelService[models2.TaskV2]().GetById(taskId)
if err != nil {
return nil, err
}
// spider
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId)
s, err := service.NewModelService[models2.SpiderV2]().GetById(t.SpiderId)
if err != nil {
return nil, err
}
@@ -137,7 +137,7 @@ func (svc *Service) getDatabaseServiceItem(taskId primitive.ObjectID) (item *dat
}
func (svc *Service) updateTaskStats(id primitive.ObjectID, resultCount int) {
err := service.NewModelServiceV2[models2.TaskStatV2]().UpdateById(id, bson.M{
err := service.NewModelService[models2.TaskStatV2]().UpdateById(id, bson.M{
"$inc": bson.M{
"result_count": resultCount,
},