Merge branch 'develop' of https://github.com/crawlab-team/crawlab into develop

backend/scripts/install-nodejs.sh

@@ -10,9 +10,8 @@ touch /tmp/install.lock
 touch /tmp/install-nodejs.lock
 
 # install node.js
-curl -sL https://deb.nodesource.com/setup_10.x | bash -
-apt-get update && apt install -y nodejs nodejs-dev node-gyp libssl1.0-dev
-apt-get update && apt install -y npm
+curl -sL https://deb.nodesource.com/setup_12.x | bash -
+apt-get update && apt install -y nodejs 
 
 # install chromium
 # See https://crbug.com/795759

backend/services/sys_exec/linux_mac.go

@@ -0,0 +1,30 @@
+// +build !windows
+
+package sys_exec
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+func BuildCmd(cmdStr string) *exec.Cmd {
+	return exec.Command("sh", "-c", cmdStr)
+}
+
+func Setpgid(cmd *exec.Cmd) {
+	if cmd == nil {
+		return
+	}
+	if cmd.SysProcAttr == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	} else {
+		cmd.SysProcAttr.Setpgid = true
+	}
+}
+
+func KillProcess(cmd *exec.Cmd) error {
+	if cmd == nil {
+		return nil
+	}
+	return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+}

backend/services/sys_exec/windows.go

@@ -0,0 +1,24 @@
+// +build windows
+
+package sys_exec
+
+import (
+	"os/exec"
+)
+
+func BuildCmd(cmdStr string) *exec.Cmd {
+	return exec.Command("cmd", "/C", cmdStr)
+}
+
+func Setpgid(cmd *exec.Cmd) {
+	return
+}
+
+func KillProcess(cmd *exec.Cmd) error {
+	if cmd != nil && cmd.Process != nil {
+		if err := cmd.Process.Kill(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

backend/services/task.go

@@ -10,6 +10,7 @@ import (
 	"crawlab/services/notification"
 	"crawlab/services/rpc"
 	"crawlab/services/spider_handler"
+	"crawlab/services/sys_exec"
 	"crawlab/utils"
 	"encoding/json"
 	"errors"
@@ -24,12 +25,10 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"runtime/debug"
 	"strconv"
 	"strings"
 	"sync"
-	"syscall"
 	"time"
 )
 
@@ -281,15 +280,8 @@ func FinishOrCancelTask(ch chan string, cmd *exec.Cmd, s model.Spider, t model.T
 	log.Infof("process received signal: %s", signal)
 
 	if signal == constants.TaskCancel && cmd.Process != nil {
-		var err error
-		// 兼容windows
-		if runtime.GOOS == constants.Windows {
-			err = cmd.Process.Kill()
-		} else {
-			err = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
-		}
-		// 取消进程
-		if err != nil {
+		// 终止进程
+		if err := sys_exec.KillProcess(cmd); err != nil {
 			log.Errorf("process kill error: %s", err.Error())
 			debug.PrintStack()
 
@@ -359,12 +351,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
 	wg := &sync.WaitGroup{}
 
 	// 生成执行命令
-	var cmd *exec.Cmd
-	if runtime.GOOS == constants.Windows {
-		cmd = exec.Command("cmd", "/C", cmdStr)
-	} else {
-		cmd = exec.Command("sh", "-c", cmdStr)
-	}
+	cmd := sys_exec.BuildCmd(cmdStr)
 
 	// 工作目录
 	cmd.Dir = cwd
@@ -395,9 +382,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
 	go FinishOrCancelTask(ch, cmd, s, t)
 
 	// kill的时候，可以kill所有的子进程
-	if runtime.GOOS != constants.Windows {
-		cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
-	}
+	sys_exec.Setpgid(cmd)
 
 	// 启动进程
 	if err := StartTaskProcess(cmd, t); err != nil {

docker_init.sh

@@ -22,6 +22,9 @@ fi
 # start nginx
 service nginx start
 
+#grant script 
+chmod +x /app/backend/scripts/*.sh
+
 # install languages
 if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_DOTNET}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ];
 then