From 10e8827dd3f716b7b954eb31d7a0f3820850e241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 14:46:03 +0800 Subject: [PATCH 1/5] =?UTF-8?q?fix=20=E9=9D=9E=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E7=88=AC=E8=99=AB=E7=9C=8B=E4=B8=8D=E5=88=B0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9A=84=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/spider/SpiderDetail.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index 916592f4..b42e750d 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -16,7 +16,7 @@ - + From 4c8b38f40bdc5b1376dd026d86a2f34b3d51dbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 19:41:29 +0800 Subject: [PATCH 2/5] =?UTF-8?q?fix=20=E7=8A=B6=E6=80=81=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/services/task.go b/backend/services/task.go index 0e8db964..6b0effe6 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -14,10 +14,12 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "runtime/debug" "strconv" "sync" + "syscall" "time" ) @@ -142,7 +144,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e log.Infof("cancel process signal: %s", signal) if signal == constants.TaskCancel && cmd.Process != nil { // 取消进程 - if err := cmd.Process.Kill(); err != nil { + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { log.Errorf("process kill error: %s", err.Error()) debug.PrintStack() } @@ -152,6 +154,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e t.Status = constants.StatusFinished } t.FinishTs = time.Now() + t.Error = "user kill the process ..." if err := t.Save(); err != nil { log.Infof("save task error: %s", err.Error()) debug.PrintStack() @@ -159,6 +162,8 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e } }() + // 在选择所有节点执行的时候，实际就是随机一个节点执行的， + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} // 异步启动进程 if err := cmd.Start(); err != nil { log.Errorf("start spider error:{}", err.Error()) @@ -177,11 +182,12 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e log.Errorf("wait process finish error: %s", err.Error()) debug.PrintStack() + log.Infof("error type is : %s", reflect.TypeOf(err).String()) // 发生一次也需要保存 - t.Error = err.Error() - t.FinishTs = time.Now() - t.Status = constants.StatusFinished - _ = t.Save() + //t.Error = err.Error() + //t.FinishTs = time.Now() + //t.Status = constants.StatusError + //_ = t.Save() return err } ch <- constants.TaskFinish From a081b02c13440a8323e51f1b3ef7165cca5e1127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 20:16:49 +0800 Subject: [PATCH 3/5] =?UTF-8?q?fix=20=E7=8A=B6=E6=80=81=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/backend/services/task.go b/backend/services/task.go index 6b0effe6..cf0f61a8 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -164,12 +164,14 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e // 在选择所有节点执行的时候，实际就是随机一个节点执行的， cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + // 异步启动进程 if err := cmd.Start(); err != nil { log.Errorf("start spider error:{}", err.Error()) debug.PrintStack() return err } + // 保存pid到task t.Pid = cmd.Process.Pid if err := t.Save(); err != nil { @@ -183,11 +185,18 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e debug.PrintStack() log.Infof("error type is : %s", reflect.TypeOf(err).String()) - // 发生一次也需要保存 - //t.Error = err.Error() - //t.FinishTs = time.Now() - //t.Status = constants.StatusError - //_ = t.Save() + if exitError, ok := err.(*exec.ExitError); ok { + exitCode := exitError.ExitCode() + log.Errorf("exit error, exit code: %d", exitCode) + // 非kill 的错误类型 + if exitCode != 9 { + // 发生一次也需要保存 + t.Error = err.Error() + t.FinishTs = time.Now() + t.Status = constants.StatusError + _ = t.Save() + } + } return err } ch <- constants.TaskFinish From 6b90d86ea28b4559d3c02a328b718e0cb9f459d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 20:20:48 +0800 Subject: [PATCH 4/5] =?UTF-8?q?fix=20=E7=8A=B6=E6=80=81=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/task.go b/backend/services/task.go index cf0f61a8..9654e8d5 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -189,7 +189,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e exitCode := exitError.ExitCode() log.Errorf("exit error, exit code: %d", exitCode) // 非kill 的错误类型 - if exitCode != 9 { + if exitCode != -1 { // 发生一次也需要保存 t.Error = err.Error() t.FinishTs = time.Now() From f5af83dccee7d8b2662bbf741ffeca040f2e5602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 20:26:08 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 5 +---- frontend/package.json | 2 +- frontend/src/views/layout/components/Navbar.vue | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/services/task.go b/backend/services/task.go index 9654e8d5..ce62a95e 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -14,7 +14,6 @@ import ( "os" "os/exec" "path/filepath" - "reflect" "runtime" "runtime/debug" "strconv" @@ -183,14 +182,12 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e if err := cmd.Wait(); err != nil { log.Errorf("wait process finish error: %s", err.Error()) debug.PrintStack() - - log.Infof("error type is : %s", reflect.TypeOf(err).String()) if exitError, ok := err.(*exec.ExitError); ok { exitCode := exitError.ExitCode() log.Errorf("exit error, exit code: %d", exitCode) // 非kill 的错误类型 if exitCode != -1 { - // 发生一次也需要保存 + // 非手动kill保存为错误状态 t.Error = err.Error() t.FinishTs = time.Now() t.Status = constants.StatusError diff --git a/frontend/package.json b/frontend/package.json index f5d170c8..60ac5cc8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "crawlab", - "version": "0.3.2", + "version": "0.3.4", "private": true, "scripts": { "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development", diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index 25d62e35..e294ad0c 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -9,7 +9,7 @@ - v0.3.2 + v0.3.4 {{$t('Logout')}}