Merge pull request #818 from crawlab-team/develop

Develop
2026-01-22 17:31:03 +01:00 · 2020-07-31 13:42:39 +08:00
parent aabedb1c6d 4991141189
commit dcb9351220
41 changed files with 871 additions and 207 deletions
--- a/CHANGELOG-zh.md
+++ b/CHANGELOG-zh.md
@@ -1,3 +1,20 @@
+# 0.5.1 (2020-07-31)
+### 功能 / 优化
+- **加入错误详情信息**.
+- **加入 Golang 编程语言支持**.
+- **加入 Chrome Driver 和 Firefox 的 Web Driver 安装脚本**.
+- **支持系统任务**. "系统任务"跟普通爬虫任务相似，允许用户查看诸如安装语言之类的任务日志.
+- **将安装语言从 RPC 更改为系统任务**.
+
+### Bug 修复
+- **修复在爬虫市场中第一次下载爬虫时会报500错误**. [#808](https://github.com/crawlab-team/crawlab/issues/808)
+- **修复一部分翻译问题**.
+- **修复任务详情 500 错误**. [#810](https://github.com/crawlab-team/crawlab/issues/810)
+- **修复密码重置问题**. [#811](https://github.com/crawlab-team/crawlab/issues/811)
+- **修复无法下载 CSV 问题**. [#812](https://github.com/crawlab-team/crawlab/issues/812)
+- **修复无法安装 Node.js 问题**. [#813](https://github.com/crawlab-team/crawlab/issues/813)
+- **修复批量添加定时任务时默认为禁用问题**. [#814](https://github.com/crawlab-team/crawlab/issues/814)
+
 # 0.5.0 (2020-07-19)
 ### 功能 / 优化
 - **爬虫市场**. 允许用户下载开源爬虫到 Crawlab.
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,20 @@
+# 0.5.1 (2020-07-31)
+### Features / Enhancement
+- **Added error message details**.
+- **Added Golang programming language support**.
+- **Added web driver installation scripts for Chrome Driver and Firefox**.
+- **Support system tasks**. A "system task" is similar to normal spider task, it allows users to view logs of general tasks such as installing languages.
+- **Changed methods of installing languages from RPC to system tasks**.
+
+### Bug Fixes
+- **Fixed first download repo 500 error in Spider Market page**. [#808](https://github.com/crawlab-team/crawlab/issues/808)
+- **Fixed some translation issues**.
+- **Fixed 500 error in task detail page**. [#810](https://github.com/crawlab-team/crawlab/issues/810)
+- **Fixed password reset issue**. [#811](https://github.com/crawlab-team/crawlab/issues/811)
+- **Fixed unable to download CSV issue**. [#812](https://github.com/crawlab-team/crawlab/issues/812)
+- **Fixed unable to install node.js issue**. [#813](https://github.com/crawlab-team/crawlab/issues/813)
+- **Fixed disabled status for batch adding schedules**. [#814](https://github.com/crawlab-team/crawlab/issues/814)
+
 # 0.5.0 (2020-07-19)
 ### Features / Enhancement
 - **Spider Market**. Allow users to download open-source spiders into Crawlab.
--- a/backend/conf/config.yml
+++ b/backend/conf/config.yml
@@ -33,13 +33,14 @@ server:
    java: "N"
    dotnet: "N"
    php: "N"
+  scripts: "/app/backend/scripts"
 spider:
  path: "/app/spiders"
 task:
  workers: 16
 other:
  tmppath: "/tmp"
-version: 0.5.0
+version: 0.5.1
 setting:
  crawlabLogToES: "N" # Send crawlab runtime log to ES, open this option "Y", remember to set esClient
  crawlabLogIndex: "crawlab-log"
--- a/backend/constants/system.go
+++ b/backend/constants/system.go
@@ -18,3 +18,8 @@ const (
 	InstallStatusInstallingOther = "installing-other"
 	InstallStatusInstalled       = "installed"
 )
+
+const (
+	LangTypeLang      = "lang"
+	LangTypeWebDriver = "webdriver"
+)
--- a/backend/constants/task.go
+++ b/backend/constants/task.go
@@ -25,3 +25,8 @@ const (
 	RunTypeRandom        string = "random"
 	RunTypeSelectedNodes string = "selected-nodes"
 )
+
+const (
+	TaskTypeSpider string = "spider"
+	TaskTypeSystem string = "system"
+)
--- a/backend/entity/system.go
+++ b/backend/entity/system.go
@@ -24,6 +24,7 @@ type Lang struct {
 	InstallStatus     string   `json:"install_status"`
 	DepFileName       string   `json:"dep_file_name"`
 	InstallDepArgs    string   `json:"install_dep_cmd"`
+	Type              string   `json:"type"`
 }

 type Dependency struct {
--- a/backend/main.go
+++ b/backend/main.go
@@ -254,6 +254,11 @@ func main() {
 				authGroup.POST("/tasks-cancel", routes.CancelSelectedTask)                  // 批量取消任务
 				authGroup.POST("/tasks-restart", routes.RestartSelectedTask)                // 批量重试任务
 			}
+			// 系统任务/脚本
+			{
+				authGroup.PUT("/system-tasks", routes.PutSystemTask)      // 运行系统任务
+				authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表
+			}
 			// 定时任务
 			{
 				authGroup.GET("/schedules", routes.GetScheduleList)                  // 定时任务列表
@@ -269,13 +274,14 @@ func main() {
 			}
 			// 用户
 			{
-				authGroup.GET("/users", routes.GetUserList)       // 用户列表
-				authGroup.GET("/users/:id", routes.GetUser)       // 用户详情
-				authGroup.POST("/users/:id", routes.PostUser)     // 更改用户
-				authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
-				authGroup.PUT("/users-add", routes.PutUser)       // 添加用户
-				authGroup.GET("/me", routes.GetMe)                // 获取自己账户
-				authGroup.POST("/me", routes.PostMe)              // 修改自己账户
+				authGroup.GET("/users", routes.GetUserList)                        // 用户列表
+				authGroup.GET("/users/:id", routes.GetUser)                        // 用户详情
+				authGroup.POST("/users/:id", routes.PostUser)                      // 更改用户
+				authGroup.DELETE("/users/:id", routes.DeleteUser)                  // 删除用户
+				authGroup.PUT("/users-add", routes.PutUser)                        // 添加用户
+				authGroup.GET("/me", routes.GetMe)                                 // 获取自己账户
+				authGroup.POST("/me", routes.PostMe)                               // 修改自己账户
+				authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码
 			}
 			// 系统
 			{
--- a/backend/model/schedule.go
+++ b/backend/model/schedule.go
@@ -69,7 +69,10 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 		if schedule.RunType == constants.RunTypeSelectedNodes {
 			for _, nodeId := range schedule.NodeIds {
 				// 选择单一节点
-				node, _ := GetNode(nodeId)
+				node, err := GetNode(nodeId)
+				if err != nil {
+					continue
+				}
 				schedule.Nodes = append(schedule.Nodes, node)
 			}
 		}
--- a/backend/model/task.go
+++ b/backend/model/task.go
@@ -5,6 +5,7 @@ import (
 	"crawlab/database"
 	"crawlab/utils"
 	"github.com/apex/log"
+	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"runtime/debug"
 	"time"
@@ -29,6 +30,7 @@ type Task struct {
 	Pid             int           `json:"pid" bson:"pid"`
 	RunType         string        `json:"run_type" bson:"run_type"`
 	ScheduleId      bson.ObjectId `json:"schedule_id" bson:"schedule_id"`
+	Type            string        `json:"type" bson:"type"`

 	// 前端数据
 	SpiderName string   `json:"spider_name"`
@@ -514,3 +516,19 @@ func UpdateTaskErrorLogs(taskId string, errorRegexPattern string) error {

 	return nil
 }
+
+func GetTaskByFilter(filter bson.M) (t Task, err error) {
+	s, c := database.GetCol("tasks")
+	defer s.Close()
+
+	if err := c.Find(filter).One(&t); err != nil {
+		if err != mgo.ErrNotFound {
+			log.Errorf("find task by filter error: " + err.Error())
+			debug.PrintStack()
+			return t, err
+		}
+		return t, err
+	}
+
+	return t, nil
+}
--- a/backend/routes/schedule.go
+++ b/backend/routes/schedule.go
@@ -242,6 +242,9 @@ func PutBatchSchedules(c *gin.Context) {
 		// 添加 UserID
 		s.UserId = services.GetCurrentUserId(c)

+		// 默认启用
+		s.Enabled = true
+
 		// 添加定时任务
 		if err := model.AddSchedule(s); err != nil {
 			log.Errorf("add schedule error: " + err.Error())
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -812,6 +812,7 @@ func RunSelectedSpider(c *gin.Context) {
 					UserId:     services.GetCurrentUserId(c),
 					RunType:    constants.RunTypeAllNodes,
 					ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+					Type:       constants.TaskTypeSpider,
 				}

 				id, err := services.AddTask(t)
@@ -830,6 +831,7 @@ func RunSelectedSpider(c *gin.Context) {
 				UserId:     services.GetCurrentUserId(c),
 				RunType:    constants.RunTypeRandom,
 				ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+				Type:       constants.TaskTypeSpider,
 			}
 			id, err := services.AddTask(t)
 			if err != nil {
@@ -847,6 +849,7 @@ func RunSelectedSpider(c *gin.Context) {
 					UserId:     services.GetCurrentUserId(c),
 					RunType:    constants.RunTypeSelectedNodes,
 					ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+					Type:       constants.TaskTypeSpider,
 				}

 				id, err := services.AddTask(t)
--- a/backend/routes/system_tasks.go
+++ b/backend/routes/system_tasks.go
@@ -0,0 +1,118 @@
+package routes
+
+import (
+	"crawlab/constants"
+	"crawlab/model"
+	"crawlab/services"
+	"crawlab/utils"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/globalsign/mgo/bson"
+	"net/http"
+)
+
+func GetSystemScripts(c *gin.Context) {
+	HandleSuccessData(c, utils.GetSystemScripts())
+}
+
+func PutSystemTask(c *gin.Context) {
+	type TaskRequestBody struct {
+		RunType string          `json:"run_type"`
+		NodeIds []bson.ObjectId `json:"node_ids"`
+		Script  string          `json:"script"`
+	}
+
+	// 绑定数据
+	var reqBody TaskRequestBody
+	if err := c.ShouldBindJSON(&reqBody); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+
+	// 校验脚本参数不为空
+	if reqBody.Script == "" {
+		HandleErrorF(http.StatusBadRequest, c, "script cannot be empty")
+		return
+	}
+
+	// 校验脚本参数是否存在
+	var allScripts = utils.GetSystemScripts()
+	if !utils.StringArrayContains(allScripts, reqBody.Script) {
+		HandleErrorF(http.StatusBadRequest, c, "script does not exist")
+		return
+	}
+
+	// TODO: 校验脚本是否正在运行
+
+	// 获取执行命令
+	cmd := fmt.Sprintf("sh %s", utils.GetSystemScriptPath(reqBody.Script))
+
+	// 任务ID
+	var taskIds []string
+
+	if reqBody.RunType == constants.RunTypeAllNodes {
+		// 所有节点
+		nodes, err := model.GetNodeList(nil)
+		if err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+		for _, node := range nodes {
+			t := model.Task{
+				SpiderId:   bson.ObjectIdHex(constants.ObjectIdNull),
+				NodeId:     node.Id,
+				UserId:     services.GetCurrentUserId(c),
+				RunType:    constants.RunTypeAllNodes,
+				ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+				Type:       constants.TaskTypeSystem,
+				Cmd:        cmd,
+			}
+			id, err := services.AddTask(t)
+			if err != nil {
+				HandleError(http.StatusInternalServerError, c, err)
+				return
+			}
+			taskIds = append(taskIds, id)
+		}
+	} else if reqBody.RunType == constants.RunTypeRandom {
+		// 随机
+		t := model.Task{
+			SpiderId:   bson.ObjectIdHex(constants.ObjectIdNull),
+			UserId:     services.GetCurrentUserId(c),
+			RunType:    constants.RunTypeRandom,
+			ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+			Type:       constants.TaskTypeSystem,
+			Cmd:        cmd,
+		}
+		id, err := services.AddTask(t)
+		if err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+		taskIds = append(taskIds, id)
+	} else if reqBody.RunType == constants.RunTypeSelectedNodes {
+		// 指定节点
+		for _, nodeId := range reqBody.NodeIds {
+			t := model.Task{
+				SpiderId:   bson.ObjectIdHex(constants.ObjectIdNull),
+				NodeId:     nodeId,
+				UserId:     services.GetCurrentUserId(c),
+				RunType:    constants.RunTypeSelectedNodes,
+				ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+				Type:       constants.TaskTypeSystem,
+				Cmd:        cmd,
+			}
+			id, err := services.AddTask(t)
+			if err != nil {
+				HandleError(http.StatusInternalServerError, c, err)
+				return
+			}
+			taskIds = append(taskIds, id)
+		}
+	} else {
+		HandleErrorF(http.StatusInternalServerError, c, "invalid run_type")
+		return
+	}
+
+	HandleSuccessData(c, taskIds)
+}
--- a/backend/routes/task.go
+++ b/backend/routes/task.go
@@ -19,6 +19,7 @@ type TaskListRequestData struct {
 	SpiderId   string `form:"spider_id"`
 	ScheduleId string `form:"schedule_id"`
 	Status     string `form:"status"`
+	Type       string `form:"type"`
 }

 type TaskResultsRequestData struct {
@@ -64,6 +65,9 @@ func GetTaskList(c *gin.Context) {
 	if data.ScheduleId != "" {
 		query["schedule_id"] = bson.ObjectIdHex(data.ScheduleId)
 	}
+	if data.Type != "" {
+		query["type"] = data.Type
+	}

 	// 获取校验
 	query = services.GetAuthQuery(query, c)
@@ -150,6 +154,7 @@ func PutTask(c *gin.Context) {
 				UserId:     services.GetCurrentUserId(c),
 				RunType:    constants.RunTypeAllNodes,
 				ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+				Type:       constants.TaskTypeSpider,
 			}

 			id, err := services.AddTask(t)
@@ -168,6 +173,7 @@ func PutTask(c *gin.Context) {
 			UserId:     services.GetCurrentUserId(c),
 			RunType:    constants.RunTypeRandom,
 			ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+			Type:       constants.TaskTypeSpider,
 		}
 		id, err := services.AddTask(t)
 		if err != nil {
@@ -185,6 +191,7 @@ func PutTask(c *gin.Context) {
 				UserId:     services.GetCurrentUserId(c),
 				RunType:    constants.RunTypeSelectedNodes,
 				ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+				Type:       constants.TaskTypeSpider,
 			}

 			id, err := services.AddTask(t)
@@ -225,6 +232,7 @@ func PutBatchTasks(c *gin.Context) {
 					UserId:     services.GetCurrentUserId(c),
 					RunType:    constants.RunTypeAllNodes,
 					ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+					Type:       constants.TaskTypeSpider,
 				}

 				id, err := services.AddTask(t)
@@ -242,6 +250,7 @@ func PutBatchTasks(c *gin.Context) {
 				UserId:     services.GetCurrentUserId(c),
 				RunType:    constants.RunTypeRandom,
 				ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+				Type:       constants.TaskTypeSpider,
 			}
 			id, err := services.AddTask(t)
 			if err != nil {
@@ -259,6 +268,7 @@ func PutBatchTasks(c *gin.Context) {
 					UserId:     services.GetCurrentUserId(c),
 					RunType:    constants.RunTypeSelectedNodes,
 					ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+					Type:       constants.TaskTypeSpider,
 				}

 				id, err := services.AddTask(t)
--- a/backend/routes/user.go
+++ b/backend/routes/user.go
@@ -279,9 +279,6 @@ func PostMe(c *gin.Context) {
 	if reqBody.Email != "" {
 		user.Email = reqBody.Email
 	}
-	if reqBody.Password != "" {
-		user.Password = utils.EncryptPassword(reqBody.Password)
-	}
 	if reqBody.Setting.NotificationTrigger != "" {
 		user.Setting.NotificationTrigger = reqBody.Setting.NotificationTrigger
 	}
@@ -311,3 +308,33 @@ func PostMe(c *gin.Context) {
 		Message: "success",
 	})
 }
+
+func PostMeChangePassword(c *gin.Context) {
+	ctx := context.WithGinContext(c)
+	user := ctx.User()
+	if user == nil {
+		ctx.FailedWithError(constants.ErrorUserNotFound, http.StatusUnauthorized)
+		return
+	}
+	var reqBody model.User
+	if err := c.ShouldBindJSON(&reqBody); err != nil {
+		HandleErrorF(http.StatusBadRequest, c, "invalid request")
+		return
+	}
+	if reqBody.Password == "" {
+		HandleErrorF(http.StatusBadRequest, c, "password is empty")
+		return
+	}
+	if user.UserId.Hex() == "" {
+		user.UserId = bson.ObjectIdHex(constants.ObjectIdNull)
+	}
+	user.Password = utils.EncryptPassword(reqBody.Password)
+	if err := user.Save(); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
--- a/backend/scripts/install-chromedriver.sh
+++ b/backend/scripts/install-chromedriver.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# fail immediately if error
+set -e
+
+# lock global
+touch /tmp/install.lock
+
+# lock
+touch /tmp/install-chromedriver.lock
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install unzip
+DL=https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+curl -sL "$DL" > /tmp/chrome.deb
+apt install --no-install-recommends --no-install-suggests -y /tmp/chrome.deb
+CHROMIUM_FLAGS='--no-sandbox --disable-dev-shm-usage'
+sed -i '${s/$/'" $CHROMIUM_FLAGS"'/}' /opt/google/chrome/google-chrome
+BASE_URL=https://chromedriver.storage.googleapis.com
+VERSION=$(curl -sL "$BASE_URL/LATEST_RELEASE")
+curl -sL "$BASE_URL/$VERSION/chromedriver_linux64.zip" -o /tmp/driver.zip
+unzip /tmp/driver.zip
+chmod 755 chromedriver
+mv chromedriver /usr/local/bin
+
+# unlock global
+rm /tmp/install.lock
+
+# unlock
+rm /tmp/install-chromedriver.lock
--- a/backend/scripts/install-dotnet.sh
+++ b/backend/scripts/install-dotnet.sh
@@ -1,3 +1,8 @@
+#!/bin/bash
+
+# fail immediately if error
+set -e
+
 # lock global
 touch /tmp/install.lock

--- a/backend/scripts/install-firefox.sh
+++ b/backend/scripts/install-firefox.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# fail immediately if error
+set -e
+
+# lock global
+touch /tmp/install.lock
+
+# lock
+touch /tmp/install-firefox.lock
+
+apt-get update
+apt-get -y install  firefox ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy
+apt-get -y install  libcanberra-gtk3-module
+
+# unlock global
+rm /tmp/install.lock
+
+# unlock
+rm /tmp/install-firefox.lock
--- a/backend/scripts/install-go.sh
+++ b/backend/scripts/install-go.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# fail immediately if error
+set -e
+
+# lock global
+touch /tmp/install.lock
+
+# lock
+touch /tmp/install-go.lock
+
+# install golang
+apt-get update
+apt-get install -y golang
+
+# environment variables
+export GOPROXY=https://goproxy.cn
+export GOPATH=/opt/go
+
+# unlock global
+rm /tmp/install.lock
+
+# unlock
+rm /tmp/install-go.lock
--- a/backend/scripts/install-java.sh
+++ b/backend/scripts/install-java.sh
@@ -1,5 +1,8 @@
 #!/bin/bash

+# fail immediately if error
+set -e
+
 # lock global
 touch /tmp/install.lock

@@ -7,9 +10,9 @@ touch /tmp/install.lock
 touch /tmp/install-java.lock

 # install java
-apt-get clean && \
-	apt-get update --fix-missing && \
-	apt-get install -y --fix-missing default-jdk
+apt-get clean
+apt-get update --fix-missing
+apt-get install -y --fix-missing default-jdk
 ln -s /usr/bin/java /usr/local/bin/java

 # unlock
--- a/backend/scripts/install-nodejs.sh
+++ b/backend/scripts/install-nodejs.sh
@@ -1,28 +1,18 @@
 #!/bin/bash

+# fail immediately if error
+set -e
+
 # lock global
 touch /tmp/install.lock

 # lock
 touch /tmp/install-nodejs.lock

-# install nvm
-BASE_DIR=`dirname $0`
-/bin/bash ${BASE_DIR}/install-nvm.sh
-export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
-
-# install Node.js v10.19
-export NVM_NODEJS_ORG_MIRROR=http://npm.taobao.org/mirrors/node
-nvm install 10.19
-
-# create soft links
-ln -s $HOME/.nvm/versions/node/v10.19.0/bin/npm /usr/local/bin/npm
-ln -s $HOME/.nvm/versions/node/v10.19.0/bin/node /usr/local/bin/node
-
-# environments manipulation
-export NODE_PATH=$HOME.nvm/versions/node/v10.19.0/lib/node_modules
-export PATH=$NODE_PATH:$PATH
+# install node.js
+curl -sL https://deb.nodesource.com/setup_10.x | bash -
+apt-get update && apt install -y nodejs nodejs-dev node-gyp libssl1.0-dev
+apt-get update && apt install -y npm

 # install chromium
 # See https://crbug.com/795759
@@ -33,7 +23,17 @@ apt-get update && apt-get install -yq libgconf-2-4
 # Note: this installs the necessary libs to make the bundled version 
 # of Chromium that Puppeteer
 # installs, work.
-apt-get update && apt-get install -y --no-install-recommends gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget
+apt-get update \
+    && apt-get install -y wget gnupg \
+    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+    && apt-get update \
+    && apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
+      libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
+      libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
+      libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget \
+    && rm -rf /var/lib/apt/lists/*

 # install default dependencies
 PUPPETEER_DOWNLOAD_HOST=https://npm.taobao.org/mirrors
--- a/backend/scripts/install-nvm.sh
+++ b/backend/scripts/install-nvm.sh
@@ -1,4 +1,7 @@
-#!/usr/bin/env bash
+#!/bin/bash
+
+# fail immediately if error
+set -e

 { # this ensures the entire script is downloaded #

--- a/backend/scripts/install-php.sh
+++ b/backend/scripts/install-php.sh
@@ -1,3 +1,8 @@
+#!/bin/bash
+
+# fail immediately if error
+set -e
+
 # lock global
 touch /tmp/install.lock

--- a/backend/scripts/install.sh
+++ b/backend/scripts/install.sh
@@ -1,5 +1,8 @@
 #!/bin/bash

+# fail immediately if error
+set -e
+
 # install node.js
 if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ];
 then
@@ -23,3 +26,19 @@ then
 	/bin/sh /app/backend/scripts/install-dotnet.sh
 	echo "installed dotnet"
 fi
+
+# install php
+if [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ];
+then
+	echo "installing php"
+	/bin/sh /app/backend/scripts/install-php.sh
+	echo "installed php"
+fi
+
+# install go
+if [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ];
+then
+	echo "installing go"
+	/bin/sh /app/backend/scripts/install-go.sh
+	echo "installed go"
+fi
--- a/backend/services/repo.go
+++ b/backend/services/repo.go
@@ -58,7 +58,7 @@ func DownloadRepo(fullName string, userId bson.ObjectId) (err error) {
 	spider := model.GetSpiderByName(spiderName)
 	if spider.Name == "" {
 		// 新增
-		spider := model.Spider{
+		spider = model.Spider{
 			Id:          bson.NewObjectId(),
 			Name:        spiderName,
 			DisplayName: spiderName,
--- a/backend/services/rpc/get_lang.go
+++ b/backend/services/rpc/get_lang.go
@@ -48,17 +48,17 @@ func GetLangLocal(lang entity.Lang) entity.Lang {
 		}
 	}

-	// 检查是否正在安装
-	if utils.Exists(lang.LockPath) {
-		lang.InstallStatus = constants.InstallStatusInstalling
-		return lang
-	}
-
-	// 检查其他语言是否在安装
-	if utils.Exists("/tmp/install.lock") {
-		lang.InstallStatus = constants.InstallStatusInstallingOther
-		return lang
-	}
+	//// 检查是否正在安装
+	//if utils.Exists(lang.LockPath) {
+	//	lang.InstallStatus = constants.InstallStatusInstalling
+	//	return lang
+	//}
+	//
+	//// 检查其他语言是否在安装
+	//if utils.Exists("/tmp/install.lock") {
+	//	lang.InstallStatus = constants.InstallStatusInstallingOther
+	//	return lang
+	//}

 	lang.InstallStatus = constants.InstallStatusNotInstalled
 	return lang
--- a/backend/services/schedule.go
+++ b/backend/services/schedule.go
@@ -58,6 +58,7 @@ func AddScheduleTask(s model.Schedule) func() {
 					UserId:     s.UserId,
 					RunType:    constants.RunTypeAllNodes,
 					ScheduleId: s.Id,
+					Type:       constants.TaskTypeSpider,
 				}

 				if _, err := AddTask(t); err != nil {
@@ -73,6 +74,7 @@ func AddScheduleTask(s model.Schedule) func() {
 				UserId:     s.UserId,
 				RunType:    constants.RunTypeRandom,
 				ScheduleId: s.Id,
+				Type:       constants.TaskTypeSpider,
 			}
 			if _, err := AddTask(t); err != nil {
 				log.Errorf(err.Error())
@@ -90,6 +92,7 @@ func AddScheduleTask(s model.Schedule) func() {
 					UserId:     s.UserId,
 					RunType:    constants.RunTypeSelectedNodes,
 					ScheduleId: s.Id,
+					Type:       constants.TaskTypeSpider,
 				}

 				if _, err := AddTask(t); err != nil {
--- a/backend/services/system.go
+++ b/backend/services/system.go
@@ -5,12 +5,15 @@ import (
 	"crawlab/database"
 	"crawlab/entity"
 	"crawlab/lib/cron"
+	"crawlab/model"
 	"crawlab/services/rpc"
 	"crawlab/utils"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"github.com/apex/log"
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
 	"github.com/imroc/req"
 	"os/exec"
 	"regexp"
@@ -71,7 +74,24 @@ func GetLangList(nodeId string) []entity.Lang {
 	return list
 }

+// 获取语言安装状态
 func GetLangInstallStatus(nodeId string, lang entity.Lang) (string, error) {
+	_, err := model.GetTaskByFilter(bson.M{
+		"node_id": nodeId,
+		"cmd":     fmt.Sprintf("sh %s", utils.GetSystemScriptPath(lang.InstallScript)),
+		"status": bson.M{
+			"$in": []string{constants.StatusPending, constants.StatusRunning},
+		},
+	})
+	if err == nil {
+		// 任务正在运行，正在安装
+		return constants.InstallStatusInstalling, nil
+	}
+	if err != mgo.ErrNotFound {
+		// 发生错误
+		return "", err
+	}
+	// 获取状态
 	if IsMasterNode(nodeId) {
 		lang := rpc.GetLangLocal(lang)
 		return lang.InstallStatus, nil
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -23,7 +23,6 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
@@ -116,9 +115,7 @@ func AssignTask(task model.Task) error {
 func SetEnv(cmd *exec.Cmd, envs []model.Env, task model.Task, spider model.Spider) *exec.Cmd {
 	// 默认把Node.js的全局node_modules加入环境变量
 	envPath := os.Getenv("PATH")
-	homePath := os.Getenv("HOME")
-	nodeVersion := "v10.19.0"
-	nodePath := path.Join(homePath, ".nvm/versions/node", nodeVersion, "lib/node_modules")
+	nodePath := "/usr/lib/node_modules"
 	if !strings.Contains(envPath, nodePath) {
 		_ = os.Setenv("PATH", nodePath+":"+envPath)
 	}
@@ -411,37 +408,17 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
 	if err := WaitTaskProcess(cmd, t, s); err != nil {
 		return err
 	}
-	ch <- constants.TaskFinish
-	return nil
-}

-// 生成日志目录
-func MakeLogDir(t model.Task) (fileDir string, err error) {
-	// 日志目录
-	fileDir = filepath.Join(viper.GetString("log.path"), t.SpiderId.Hex())
-
-	// 如果日志目录不存在，生成该目录
-	if !utils.Exists(fileDir) {
-		if err := os.MkdirAll(fileDir, 0777); err != nil {
-			log.Errorf("execute task, make log dir error: %s", err.Error())
-			debug.PrintStack()
-			return "", err
-		}
+	// 如果返回值不为0，返回错误
+	returnCode := cmd.ProcessState.ExitCode()
+	if returnCode != 0 {
+		log.Errorf(fmt.Sprintf("task returned code not zero: %d", returnCode))
+		debug.PrintStack()
+		return errors.New(fmt.Sprintf("task returned code not zero: %d", returnCode))
 	}

-	return fileDir, nil
-}
-
-// 获取日志文件路径
-func GetLogFilePaths(fileDir string, t model.Task) (filePath string) {
-	// 时间戳
-	ts := time.Now()
-	tsStr := ts.Format("20060102150405")
-
-	// stdout日志文件
-	filePath = filepath.Join(fileDir, t.Id+"_"+tsStr+".log")
-
-	return filePath
+	ch <- constants.TaskFinish
+	return nil
 }

 // 生成执行任务方法
@@ -545,20 +522,15 @@ func ExecuteTask(id int) {
 	}

 	// 获取爬虫
-	spider, err := t.GetSpider()
-	if err != nil {
-		log.Errorf("execute task, get spider error: %s", err.Error())
-		return
+	var spider model.Spider
+	if t.Type == constants.TaskTypeSpider {
+		spider, err = t.GetSpider()
+		if err != nil {
+			log.Errorf("execute task, get spider error: %s", err.Error())
+			return
+		}
 	}

-	// 创建日志目录
-	var fileDir string
-	if fileDir, err = MakeLogDir(t); err != nil {
-		return
-	}
-	// 获取日志文件路径
-	t.LogPath = GetLogFilePaths(fileDir, t)
-
 	// 工作目录
 	cwd := filepath.Join(
 		viper.GetString("spider.path"),
@@ -567,12 +539,19 @@ func ExecuteTask(id int) {

 	// 执行命令
 	var cmd string
-	if spider.Type == constants.Configurable {
-		// 可配置爬虫命令
-		cmd = "scrapy crawl config_spider"
-	} else {
-		// 自定义爬虫命令
-		cmd = spider.Cmd
+	if t.Type == constants.TaskTypeSpider {
+		// 爬虫任务
+		if spider.Type == constants.Configurable {
+			// 可配置爬虫命令
+			cmd = "scrapy crawl config_spider"
+		} else {
+			// 自定义爬虫命令
+			cmd = spider.Cmd
+		}
+		t.Cmd = cmd
+	} else if t.Type == constants.TaskTypeSystem {
+		// 系统任务
+		cmd = t.Cmd
 	}

 	// 加入参数
@@ -593,48 +572,51 @@ func ExecuteTask(id int) {
 	t.Status = constants.StatusRunning                   // 任务状态
 	t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长

-	// 发送 Web Hook 请求 (任务开始)
-	go SendWebHookRequest(user, t, spider)
-
-	// 文件检查
-	if err := SpiderFileCheck(t, spider); err != nil {
-		log.Errorf("spider file check error: %s", err.Error())
-		return
-	}
-
-	// 开始执行任务
-	log.Infof(GetWorkerPrefix(id) + "start task (id:" + t.Id + ")")
-
 	// 储存任务
 	_ = t.Save()

-	// 创建结果集索引
-	go func() {
-		col := utils.GetSpiderCol(spider.Col, spider.Name)
-		CreateResultsIndexes(col)
-	}()
+	// 发送 Web Hook 请求 (任务开始)
+	go SendWebHookRequest(user, t, spider)

-	// 起一个cron执行器来统计任务结果数
-	cronExec := cron.New(cron.WithSeconds())
-	_, err = cronExec.AddFunc("*/5 * * * * *", SaveTaskResultCount(t.Id))
-	if err != nil {
-		log.Errorf(GetWorkerPrefix(id) + err.Error())
-		debug.PrintStack()
-		return
-	}
-	cronExec.Start()
-	defer cronExec.Stop()
+	// 爬虫任务专属逻辑
+	if t.Type == constants.TaskTypeSpider {
+		// 文件检查
+		if err := SpiderFileCheck(t, spider); err != nil {
+			log.Errorf("spider file check error: %s", err.Error())
+			return
+		}

-	// 起一个cron来更新错误日志
-	cronExecErrLog := cron.New(cron.WithSeconds())
-	_, err = cronExecErrLog.AddFunc("*/30 * * * * *", ScanErrorLogs(t))
-	if err != nil {
-		log.Errorf(GetWorkerPrefix(id) + err.Error())
-		debug.PrintStack()
-		return
+		// 开始执行任务
+		log.Infof(GetWorkerPrefix(id) + "start task (id:" + t.Id + ")")
+
+		// 创建结果集索引
+		go func() {
+			col := utils.GetSpiderCol(spider.Col, spider.Name)
+			CreateResultsIndexes(col)
+		}()
+
+		// 起一个cron执行器来统计任务结果数
+		cronExec := cron.New(cron.WithSeconds())
+		_, err = cronExec.AddFunc("*/5 * * * * *", SaveTaskResultCount(t.Id))
+		if err != nil {
+			log.Errorf(GetWorkerPrefix(id) + err.Error())
+			debug.PrintStack()
+			return
+		}
+		cronExec.Start()
+		defer cronExec.Stop()
+
+		// 起一个cron来更新错误日志
+		cronExecErrLog := cron.New(cron.WithSeconds())
+		_, err = cronExecErrLog.AddFunc("*/30 * * * * *", ScanErrorLogs(t))
+		if err != nil {
+			log.Errorf(GetWorkerPrefix(id) + err.Error())
+			debug.PrintStack()
+			return
+		}
+		cronExecErrLog.Start()
+		defer cronExecErrLog.Stop()
 	}
-	cronExecErrLog.Start()
-	defer cronExecErrLog.Stop()

 	// 执行Shell命令
 	if err := ExecuteShellCmd(cmd, cwd, t, spider, user); err != nil {
@@ -693,11 +675,13 @@ func ExecuteTask(id int) {

 func FinishUpTask(s model.Spider, t model.Task) {
 	// 更新任务结果数
-	go func() {
-		if err := model.UpdateTaskResultCount(t.Id); err != nil {
-			return
-		}
-	}()
+	if t.Type == constants.TaskTypeSpider {
+		go func() {
+			if err := model.UpdateTaskResultCount(t.Id); err != nil {
+				return
+			}
+		}()
+	}

 	// 更新任务错误日志
 	go func() {
@@ -812,10 +796,12 @@ func RestartTask(id string, uid bson.ObjectId) (err error) {
 	newTask := model.Task{
 		SpiderId:   oldTask.SpiderId,
 		NodeId:     oldTask.NodeId,
+		Cmd:        oldTask.Cmd,
 		Param:      oldTask.Param,
 		UserId:     uid,
 		RunType:    oldTask.RunType,
 		ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull),
+		Type:       oldTask.Type,
 	}

 	// 加入任务队列
--- a/backend/utils/system.go
+++ b/backend/utils/system.go
@@ -1,15 +1,20 @@
 package utils

 import (
+	"crawlab/constants"
 	"crawlab/entity"
 	"encoding/json"
 	"github.com/apex/log"
+	"github.com/spf13/viper"
 	"io/ioutil"
+	"path"
 	"runtime/debug"
+	"strings"
 )

 func GetLangList() []entity.Lang {
 	list := []entity.Lang{
+		// 语言
 		{
 			Name:              "Python",
 			ExecutableName:    "python",
@@ -18,6 +23,7 @@ func GetLangList() []entity.Lang {
 			LockPath:          "/tmp/install-python.lock",
 			DepFileName:       "requirements.txt",
 			InstallDepArgs:    "install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt",
+			Type:              constants.LangTypeLang,
 		},
 		{
 			Name:              "Node.js",
@@ -28,6 +34,7 @@ func GetLangList() []entity.Lang {
 			InstallScript:     "install-nodejs.sh",
 			DepFileName:       "package.json",
 			InstallDepArgs:    "install -g --registry=https://registry.npm.taobao.org",
+			Type:              constants.LangTypeLang,
 		},
 		{
 			Name:            "Java",
@@ -35,6 +42,7 @@ func GetLangList() []entity.Lang {
 			ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"},
 			LockPath:        "/tmp/install-java.lock",
 			InstallScript:   "install-java.sh",
+			Type:            constants.LangTypeLang,
 		},
 		{
 			Name:            ".Net Core",
@@ -42,6 +50,7 @@ func GetLangList() []entity.Lang {
 			ExecutablePaths: []string{"/usr/bin/dotnet", "/usr/local/bin/dotnet"},
 			LockPath:        "/tmp/install-dotnet.lock",
 			InstallScript:   "install-dotnet.sh",
+			Type:            constants.LangTypeLang,
 		},
 		{
 			Name:            "PHP",
@@ -49,6 +58,32 @@ func GetLangList() []entity.Lang {
 			ExecutablePaths: []string{"/usr/bin/php", "/usr/local/bin/php"},
 			LockPath:        "/tmp/install-php.lock",
 			InstallScript:   "install-php.sh",
+			Type:            constants.LangTypeLang,
+		},
+		{
+			Name:            "Golang",
+			ExecutableName:  "go",
+			ExecutablePaths: []string{"/usr/bin/go", "/usr/local/bin/go"},
+			LockPath:        "/tmp/install-go.lock",
+			InstallScript:   "install-go.sh",
+			Type:            constants.LangTypeLang,
+		},
+		// WebDriver
+		{
+			Name:            "Chrome Driver",
+			ExecutableName:  "chromedriver",
+			ExecutablePaths: []string{"/usr/bin/chromedriver", "/usr/local/bin/chromedriver"},
+			LockPath:        "/tmp/install-chromedriver.lock",
+			InstallScript:   "install-chromedriver.sh",
+			Type:            constants.LangTypeWebDriver,
+		},
+		{
+			Name:            "Firefox",
+			ExecutableName:  "firefox",
+			ExecutablePaths: []string{"/usr/bin/firefox", "/usr/local/bin/firefox"},
+			LockPath:        "/tmp/install-firefox.lock",
+			InstallScript:   "install-firefox.sh",
+			Type:            constants.LangTypeWebDriver,
 		},
 	}
 	return list
@@ -91,3 +126,24 @@ func GetPackageJsonDeps(filepath string) (deps []string, err error) {

 	return deps, nil
 }
+
+// 获取系统脚本列表
+func GetSystemScripts() (res []string) {
+	scriptsPath := viper.GetString("server.scripts")
+	for _, fInfo := range ListDir(scriptsPath) {
+		if !fInfo.IsDir() && strings.HasSuffix(fInfo.Name(), ".sh") {
+			res = append(res, fInfo.Name())
+		}
+	}
+	return res
+}
+
+func GetSystemScriptPath(scriptName string) string {
+	scriptsPath := viper.GetString("server.scripts")
+	for _, name := range GetSystemScripts() {
+		if name == scriptName {
+			return path.Join(scriptsPath, name)
+		}
+	}
+	return ""
+}
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -26,6 +26,7 @@ services:
      # CRAWLAB_SERVER_LANG_JAVA: "Y"  # whether to pre-install Java 预安装 Java 语言环境
      # CRAWLAB_SERVER_LANG_DOTNET: "Y"  # whether to pre-install .Net core 预安装 .Net Core 语言环境
      # CRAWLAB_SERVER_LANG_PHP: "Y"  # whether to pre-install PHP 预安装 PHP 语言环境
+      # CRAWLAB_SERVER_LANG_GO: "Y"  # whether to pre-install Golang 预安装 Golang 语言环境
      # CRAWLAB_SETTING_ALLOWREGISTER: "N"  # whether to allow user registration 是否允许用户注册
      # CRAWLAB_SETTING_ENABLETUTORIAL: "N"  # whether to enable tutorial 是否启用教程
      # CRAWLAB_SETTING_RUNONMASTER: "N"  # whether to run on master node 是否在主节点上运行任务
--- a/docker_init.sh
+++ b/docker_init.sh
@@ -23,7 +23,7 @@ fi
 service nginx start

 # install languages
-if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ];
+if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_DOTNET}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ];
 then
 	echo "installing languages"
 	echo "you can view log at /var/log/install.sh.log"
--- a/frontend/src/components/InfoView/TaskInfoView.vue
+++ b/frontend/src/components/InfoView/TaskInfoView.vue
@@ -24,7 +24,7 @@
            </el-tag>
          </el-badge>
          <el-tag
-            v-if="taskForm.status === 'finished' && taskForm.result_count === 0"
+            v-if="taskForm.type === 'spider' && taskForm.status === 'finished' && taskForm.result_count === 0"
            type="danger"
            style="margin-left: 10px"
          >
@@ -32,8 +32,8 @@
            {{ $t('Empty results') }}
          </el-tag>
        </el-form-item>
-        <el-form-item :label="$t('Log File Path')">
-          <el-input v-model="taskForm.log_path" placeholder="Log File Path" disabled />
+        <el-form-item :label="$t('Execute Command')">
+          <el-input v-model="taskForm.cmd" placeholder="Execute Command" disabled />
        </el-form-item>
        <el-form-item :label="$t('Parameters')">
          <el-input v-model="taskForm.param" placeholder="Parameters" disabled />
--- a/frontend/src/components/Node/NodeInstallationMatrix.vue
+++ b/frontend/src/components/Node/NodeInstallationMatrix.vue
@@ -72,7 +72,11 @@
                      <i class="el-icon-error" />
                      {{ $t('Not Installed') }}
                    </el-tag>
-                    <el-button type="primary" size="mini" @click="onInstallLang(scope.row._id, scope.column.label, $event)">
+                    <el-button
+                      type="primary"
+                      size="mini"
+                      @click="onInstallLang(scope.row._id, scope.column.label, $event)"
+                    >
                      {{ $t('Install') }}
                    </el-button>
                  </div>
@@ -203,6 +207,97 @@
          </el-table-column>
        </el-table>
      </el-tab-pane>
+      <el-tab-pane label="Web Driver" name="webdriver">
+        <div class="webdriver-table">
+          <el-table
+            class="table"
+            :data="nodeList"
+            :header-cell-style="{background:'rgb(48, 65, 86)',color:'white',height:'50px'}"
+            border
+            @row-click="onLangTableRowClick"
+          >
+            <el-table-column
+              :label="$t('Node')"
+              width="240px"
+              prop="name"
+              fixed
+            />
+            <el-table-column
+              :label="$t('nodeList.type')"
+              width="120px"
+              fixed
+            >
+              <template slot-scope="scope">
+                <el-tag v-if="scope.row.is_master" type="primary">{{ $t('Master') }}</el-tag>
+                <el-tag v-else type="warning">{{ $t('Worker') }}</el-tag>
+              </template>
+            </el-table-column>
+            <el-table-column
+              :label="$t('Status')"
+              width="120px"
+              fixed
+            >
+              <template slot-scope="scope">
+                <el-tag v-if="scope.row.status === 'offline'" type="info">{{ $t('Offline') }}</el-tag>
+                <el-tag v-else-if="scope.row.status === 'online'" type="success">{{ $t('Online') }}</el-tag>
+                <el-tag v-else type="danger">{{ $t('Unavailable') }}</el-tag>
+              </template>
+            </el-table-column>
+            <el-table-column
+              v-for="wd in webdrivers"
+              :key="wd.name"
+              :label="wd.label"
+              width="220px"
+            >
+              <template slot="header" slot-scope="scope">
+                <div class="header-with-action">
+                  <span>{{ scope.column.label }}</span>
+                  <el-button type="primary" size="mini" @click="onInstallLangAll(scope.column.label, $event)">
+                    {{ $t('Install') }}
+                  </el-button>
+                </div>
+              </template>
+              <template slot-scope="scope">
+                <template v-if="getLangInstallStatus(scope.row._id, wd.name) === 'installed'">
+                  <el-tag type="success">
+                    <i class="el-icon-check" />
+                    {{ $t('Installed') }}
+                  </el-tag>
+                </template>
+                <template v-else-if="getLangInstallStatus(scope.row._id, wd.name) === 'installing'">
+                  <el-tag type="warning">
+                    <i class="el-icon-loading" />
+                    {{ $t('Installing') }}
+                  </el-tag>
+                </template>
+                <template
+                  v-else-if="['installing-other', 'not-installed'].includes(getLangInstallStatus(scope.row._id, wd.name))"
+                >
+                  <div class="cell-with-action">
+                    <el-tag type="danger">
+                      <i class="el-icon-error" />
+                      {{ $t('Not Installed') }}
+                    </el-tag>
+                    <el-button
+                      type="primary"
+                      size="mini"
+                      @click="onInstallLang(scope.row._id, scope.column.label, $event)"
+                    >
+                      {{ $t('Install') }}
+                    </el-button>
+                  </div>
+                </template>
+                <template v-else-if="getLangInstallStatus(scope.row._id, wd.name) === 'na'">
+                  <el-tag type="info">
+                    <i class="el-icon-question" />
+                    {{ $t('N/A') }}
+                  </el-tag>
+                </template>
+              </template>
+            </el-table-column>
+          </el-table>
+        </div>
+      </el-tab-pane>
    </el-tabs>
  </div>
 </template>
@@ -220,12 +315,17 @@
    },
    data() {
      return {
-        langs: [
-          { label: 'Python', name: 'python', hasDeps: true },
-          { label: 'Node.js', name: 'node', hasDeps: true },
-          { label: 'Java', name: 'java', hasDeps: false },
-          { label: '.Net Core', name: 'dotnet', hasDeps: false },
-          { label: 'PHP', name: 'php', hasDeps: false }
+        allLangs: [
+          // 语言
+          { label: 'Python', name: 'python', hasDeps: true, script: 'install-python.sh', type: 'lang' },
+          { label: 'Node.js', name: 'node', hasDeps: true, script: 'install-nodejs.sh', type: 'lang' },
+          { label: 'Java', name: 'java', hasDeps: false, script: 'install-java.sh', type: 'lang' },
+          { label: '.Net Core', name: 'dotnet', hasDeps: false, script: 'install-dotnet.sh', type: 'lang' },
+          { label: 'PHP', name: 'php', hasDeps: false, script: 'install-php.sh', type: 'lang' },
+          { label: 'Golang', name: 'go', hasDeps: false, script: 'install-go.sh', type: 'lang' },
+          // web driver
+          { label: 'Chrome Driver', name: 'chromedriver', script: 'install-chromedriver.sh', type: 'webdriver' },
+          { label: 'Firefox', name: 'firefox', script: 'install-firefox.sh', type: 'webdriver' }
        ],
        langsDataDict: {},
        handle: undefined,
@@ -243,6 +343,12 @@
      ...mapState('node', [
        'nodeList'
      ]),
+      langs() {
+        return this.allLangs.filter(d => d.type === 'lang')
+      },
+      webdrivers() {
+        return this.allLangs.filter(d => d.type === 'webdriver')
+      },
      activeNodes() {
        return this.nodeList.filter(d => d.status === 'online')
      },
@@ -254,7 +360,7 @@
        })
      },
      langsWithDeps() {
-        return this.langs.filter(l => l.hasDeps)
+        return this.allLangs.filter(l => l.hasDeps)
      }
    },
    watch: {
@@ -315,8 +421,8 @@
        return lang.install_status
      },
      getLangFromLabel(label) {
-        for (let i = 0; i < this.langs.length; i++) {
-          const lang = this.langs[i]
+        for (let i = 0; i < this.allLangs.length; i++) {
+          const lang = this.allLangs[i]
          if (lang.label === label) {
            return lang
          }
@@ -327,17 +433,19 @@
          ev.stopPropagation()
        }
        const lang = this.getLangFromLabel(langLabel)
-        this.$request.post(`/nodes/${nodeId}/langs/install`, {
-          lang: lang.name
+        const res = await this.$request.put('/system-tasks', {
+          run_type: 'selected-nodes',
+          node_ids: [nodeId],
+          script: lang.script
        })
+        if (res && res.data && !res.data.error) {
+          this.$message.success(this.$t('Started to install') + ' ' + lang.label)
+        }
        const key = nodeId + '|' + lang.name
        this.$set(this.langsDataDict[key], 'install_status', 'installing')
        setTimeout(() => {
          this.getLangsData()
        }, 1000)
-        this.$request.put('/actions', {
-          type: 'install_lang'
-        })
        this.$st.sendEv('节点列表', '安装', '安装语言')
      },
      async onInstallLangAll(langLabel, ev) {
--- a/frontend/src/components/Overview/TaskOverview.vue
+++ b/frontend/src/components/Overview/TaskOverview.vue
@@ -2,6 +2,7 @@
  <div class="task-overview">
    <el-row class="action-wrapper">
      <el-button
+        v-if="taskForm.type === 'spider'"
        type="primary"
        size="small"
        icon="el-icon-position"
@@ -28,7 +29,7 @@
      </el-col>

      <el-col :span="12">
-        <el-row class="task-info-spider-wrapper wrapper">
+        <el-row v-if="taskForm.type === 'spider'" class="task-info-spider-wrapper wrapper">
          <h4 class="title spider-title" @click="onNavigateToSpider">
            <i class="fa fa-search" style="margin-right: 5px" />
            {{ $t('Spider Info') }}</h4>
@@ -66,6 +67,9 @@
      ]),
      ...mapState('spider', [
        'spiderForm'
+      ]),
+      ...mapState('task', [
+        'taskForm'
      ])
    },
    created() {
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -38,6 +38,7 @@ export default {
  Running: '进行中',
  Finished: '已完成',
  Error: '错误',
+  Errors: '错误',
  NA: '未知',
  Cancelled: '已取消',
  Abnormal: '异常',
@@ -104,6 +105,9 @@ export default {
  'Worker': '工作节点',
  'Installation': '安装',
  'Search Dependencies': '搜索依赖',
+  'Monitor': '监控',
+  'Time Range': '时间区间',
+  'Started to install': '开始安装',

  // 节点列表
  'IP': 'IP地址',
@@ -114,6 +118,38 @@ export default {
  Offline: '离线',
  Unavailable: '未知',

+  // 监控指标
+  'node_stats_cpu_usage_percent': '节点 CPU 使用百分比',
+  'node_stats_disk_total': '节点总磁盘大小',
+  'node_stats_disk_usage': '节点磁盘使用量',
+  'node_stats_disk_usage_percent': '节点磁盘使用百分比',
+  'node_stats_mem_total': '节点总内存大小',
+  'node_stats_mem_usage': '节点内存使用量',
+  'node_stats_mem_usage_percent': '节点内存使用百分比',
+  'node_stats_network_bytes_recv': '节点网络接收字节数',
+  'node_stats_network_bytes_sent': '节点网络发送字节数',
+  'node_stats_network_packets_recv': '节点网络接收包数',
+  'node_stats_network_packets_sent': '节点网络发送包数',
+  'mongo_stats_mem_resident': 'MongoDB 内存使用量',
+  'mongo_stats_mem_virtual': 'MongoDB 虚拟内存大小',
+  'mongo_stats_mem_usage_percent': 'MongoDB 内存使用百分比',
+  'mongo_stats_fs_total': 'MongoDB 总文件系统大小',
+  'mongo_stats_fs_used': 'MongoDB 文件系统使用量',
+  'mongo_stats_fs_usage_percent': 'MongoDB 文件系统使用百分比',
+  'mongo_stats_storage_size': 'MongoDB 储存大小',
+  'mongo_stats_data_size': 'MongoDB 数据大小',
+  'mongo_stats_index_size': 'MongoDB 索引大小',
+  'mongo_stats_objects': 'MongoDB Object 数量',
+  'mongo_stats_collections': 'MongoDB Collection 数量',
+  'mongo_stats_indexes': 'MongoDB 索引数量',
+  'mongo_stats_avg_obj_size': 'MongoDB 平均 Object 大小',
+  'redis_stats_dataset_bytes': 'Redis 数据字节数',
+  'redis_stats_keys_count': 'Redis Key 数量',
+  'redis_stats_overhead_total': 'Redis Overhead 总大小',
+  'redis_stats_peak_allocated': 'Redis 峰值分配大小',
+  'redis_stats_startup_allocated': 'Redis 启动分配大小',
+  'redis_stats_total_allocated': 'Redis 总分配大小',
+
  // 爬虫
  'Spider Info': '爬虫信息',
  'Spider ID': '爬虫ID',
@@ -121,6 +157,8 @@ export default {
  'Source Folder': '代码目录',
  'Execute Command': '执行命令',
  'Results Collection': '结果集',
+  'Results Table': '结果表',
+  'Default': '默认',
  'Spider Type': '爬虫类型',
  'Language': '语言',
  'Schedule Enabled': '是否开启定时任务',
@@ -284,6 +322,9 @@ export default {
  'Start Time': '开始时间',
  'Finish Time': '结束时间',
  'Update Time': '更新时间',
+  'Type': '类别',
+  'Spider Tasks': '爬虫任务',
+  'System Tasks': '系统任务',

  // 部署
  'Time': '时间',
@@ -425,6 +466,8 @@ export default {
  'Disclaimer': '免责声明',
  'Please search dependencies': '请搜索依赖',
  'No Data': '暂无数据',
+  'No data available': '暂无数据',
+  'No data available. Please check whether your spiders are missing dependencies or no spiders created.': '暂无数据。请检查您的爬虫是否缺少依赖，或者没有创建爬虫。',
  'Show installed': '查看已安装',
  'Installing dependency successful': '安装依赖成功',
  'Installing dependency failed': '安装依赖失败',
@@ -502,6 +545,14 @@ export default {
  'Log Errors': '日志错误',
  'No Expire': '不过期',
  'Log Expire Duration': '日志过期时间',
+  'Database': '数据库',
+  'Data Source': '数据源',
+  'Data Source Type': '数据源类别',
+  'Host': '主机',
+  'Host address, e.g. 192.168.0.1': '主机地址，例如 192.168.0.1',
+  'Port, e.g. 27017': '端口，例如 27017',
+  'Auth Source (Default: admin)': 'Auth Source (默认: admin)',
+  'Change Password': '更改密码',

  // 挑战
  'Challenge': '挑战',
@@ -556,12 +607,21 @@ export default {
    // Cron Format: [second] [minute] [hour] [day of month] [month] [day of week]
    cron_format: 'Cron 格式: [秒] [分] [小时] [日] [月] [周]'
  },
-  auth: {
-    login_expired_message: '您已注销，可以取消以保留在该页面上，或者再次登录',
-    login_expired_title: '确认登出',
-    login_expired_confirm: '确认',
-    login_expired_cancel: '取消'
-  },
+
+  // 监控
+  'Disk': '磁盘',
+  'Data Size': '数据大小',
+  'Storage Size': '储存大小',
+  'Memory': '内存',
+  'CPU': 'CPU',
+  'Index Size': '索引大小',
+  'Total Allocated': '总分配内存',
+  'Peak Allocated': '峰值内存',
+  'Dataset Size': '数据大小',
+  'Overhead Size': '额外开销',
+  'Disk Usage': '磁盘使用量',
+  'Memory Usage': '内存使用量',
+
  // 内容
  addNodeInstruction: `
 您不能在 Crawlab 的 Web 界面直接添加节点。
@@ -665,6 +725,9 @@ export default {
  'Are you sure to add an API token?': '确认创建 API Token?',
  'Are you sure to delete this API token?': '确认删除该 API Token?',
  'Please enter Web Hook URL': '请输入 Web Hook URL',
+  'Change data source failed': '更改数据源失败',
+  'Changed data source successfully': '更改数据源成功',
+  'Are you sure to delete this data source?': '您确定删除该数据源?',
  'Are you sure to download this spider?': '您确定要下载该爬虫?',
  'Downloaded successfully': '下载成功',
  'Unable to submit because of some errors': '有错误，无法提交',
@@ -676,7 +739,11 @@ export default {
  'Are you sure to stop this task?': '确认停止这个任务?',
  'Enabled successfully': '成功启用',
  'Disabled successfully': '成功禁用',
+  'Request Error': '请求错误',
+  'Changed password successfully': '成功修改密码',
+  'Two passwords do not match': '两次密码不匹配',

  // 其他
-  'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'
+  'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧',
+  'How to buy': '如何购买'
 }
--- a/frontend/src/store/modules/task.js
+++ b/frontend/src/store/modules/task.js
@@ -13,7 +13,8 @@ const state = {
    node_id: '',
    spider_id: '',
    status: '',
-    schedule_id: ''
+    schedule_id: '',
+    type: 'spider'
  },
  // pagination
  pageNum: 1,
@@ -161,7 +162,9 @@ const actions = {
      .then(response => {
        const data = response.data.data
        commit('SET_TASK_FORM', data)
-        dispatch('spider/getSpiderData', data.spider_id, { root: true })
+        if (data.type === 'spider') {
+          dispatch('spider/getSpiderData', data.spider_id, { root: true })
+        }
        if (data.node_id && data.node_id !== '000000000000000000000000') {
          dispatch('node/getNodeData', data.node_id, { root: true })
        }
@@ -174,7 +177,8 @@ const actions = {
      node_id: state.filter.node_id || undefined,
      spider_id: state.filter.spider_id || undefined,
      status: state.filter.status || undefined,
-      schedule_id: state.filter.schedule_id || undefined
+      schedule_id: state.filter.schedule_id || undefined,
+      type: state.filter.type || undefined
    })
      .then(response => {
        commit('SET_TASK_LIST', response.data.data || [])
@@ -234,10 +238,9 @@ const actions = {
      })
  },
  async getTaskResultExcel({ state, commit }, id) {
-    const { data } = await request.request('GET',
-      '/tasks/' + id + '/results/download', {}, {
-        responseType: 'blob' // important
-      })
+    const { data } = await request.get('/tasks/' + id + '/results/download', {}, {
+      responseType: 'blob' // important
+    })
    const downloadUrl = window.URL.createObjectURL(new Blob([data]))

    const link = document.createElement('a')
--- a/frontend/src/utils/request.js
+++ b/frontend/src/utils/request.js
@@ -5,23 +5,23 @@ import { getToken } from '@/utils/auth'
 import i18n from '@/i18n'
 import router from '@/router'

-const codeMessage = {
-  200: '服务器成功返回请求的数据。',
-  201: '新建或修改数据成功。',
-  202: '一个请求已经进入后台排队（异步任务）。',
-  204: '删除数据成功。',
-  400: '发出的请求有错误，服务器没有进行新建或修改数据的操作。',
-  401: '用户没有权限（令牌、用户名、密码错误）。',
-  403: '用户得到授权，但是访问是被禁止的。',
-  404: '发出的请求针对的是不存在的记录，服务器没有进行操作。',
-  406: '请求的格式不可得。',
-  410: '请求的资源被永久删除，且不会再得到的。',
-  422: '当创建一个对象时，发生一个验证错误。',
-  500: '服务器发生错误，请检查服务器。',
-  502: '网关错误。',
-  503: '服务不可用，服务器暂时过载或维护。',
-  504: '网关超时。'
-}
+// const codeMessage = {
+//   200: '服务器成功返回请求的数据。',
+//   201: '新建或修改数据成功。',
+//   202: '一个请求已经进入后台排队（异步任务）。',
+//   204: '删除数据成功。',
+//   400: '发出的请求有错误，服务器没有进行新建或修改数据的操作。',
+//   401: '用户没有权限（令牌、用户名、密码错误）。',
+//   403: '用户得到授权，但是访问是被禁止的。',
+//   404: '发出的请求针对的是不存在的记录，服务器没有进行操作。',
+//   406: '请求的格式不可得。',
+//   410: '请求的资源被永久删除，且不会再得到的。',
+//   422: '当创建一个对象时，发生一个验证错误。',
+//   500: '服务器发生错误，请检查服务器。',
+//   502: '网关错误。',
+//   503: '服务不可用，服务器暂时过载或维护。',
+//   504: '网关超时。'
+// }

 /**
 * 异常处理程序
@@ -30,10 +30,10 @@ const errorHandler = (error) => {
  const { response } = error
  const routePath = router.currentRoute.path
  if (response && response.status) {
-    const errorText = codeMessage[response.status] || response.statusText
+    const errorText = response.data.error
    const { status } = response
    Message({
-      message: `请求错误 ${status}: ${response.request.responseURL},${errorText}`,
+      message: i18n.t('Request Error') + ` ${status}: ${response.request.responseURL}. ${errorText}`,
      type: 'error',
      duration: 5 * 1000
    })
--- a/frontend/src/views/schedule/ScheduleList.vue
+++ b/frontend/src/views/schedule/ScheduleList.vue
@@ -688,7 +688,7 @@
              ids: this.selectedSchedules.map(d => d._id)
            })
            if (!res.data.error) {
-              this.$message.success('Deleted successfully')
+              this.$message.success(this.$t('Deleted successfully'))
              this.$refs['table'].clearSelection()
              await this.$store.dispatch('schedule/getScheduleList')
            }
@@ -844,7 +844,7 @@
  }

  .table {
-    min-height: 360px;
+    min-height: 720px;
    margin-top: 10px;
  }

--- a/frontend/src/views/setting/Setting.vue
+++ b/frontend/src/views/setting/Setting.vue
@@ -48,9 +48,6 @@
          <el-form-item prop="username" :label="$t('Username')">
            <el-input v-model="userInfo.username" disabled />
          </el-form-item>
-          <el-form-item prop="password" :label="$t('Password')">
-            <el-input v-model="userInfo.password" type="password" :placeholder="$t('Password')" />
-          </el-form-item>
          <el-form-item :label="$t('Allow Sending Statistics')">
            <el-switch
              v-model="isAllowSendingStatistics"
@@ -78,6 +75,32 @@
      </el-tab-pane>
      <!--./通用-->

+      <!--更改密码-->
+      <el-tab-pane :label="$t('Change Password')" name="change-password">
+        <el-form
+          ref="change-password-form"
+          :model="userInfo"
+          class="change-password-form"
+          label-width="200px"
+          inline-message
+        >
+          <el-form-item prop="password" :label="$t('Password')" required>
+            <el-input v-model="userInfo.password" type="password" :placeholder="$t('Password')" />
+          </el-form-item>
+          <el-form-item prop="confirm_password" :label="$t('Confirm Password')" required>
+            <el-input v-model="userInfo.confirm_password" type="password" :placeholder="$t('Confirm Password')" />
+          </el-form-item>
+          <el-form-item>
+            <div style="text-align: right">
+              <el-button type="success" size="small" @click="changePassword">
+                {{ $t('Save') }}
+              </el-button>
+            </div>
+          </el-form-item>
+        </el-form>
+      </el-tab-pane>
+      <!--./更改密码-->
+
      <!--消息通知-->
      <el-tab-pane :label="$t('Notifications')" name="notify">
        <el-form
@@ -517,28 +540,51 @@
        input.select()
        document.execCommand('copy')
        this.$message.success(this.$t('Token copied'))
+      },
+      changePassword() {
+        this.$refs['change-password-form'].validate(async valid => {
+          if (!valid) return
+          if (this.userInfo.password !== this.userInfo.confirm_password) {
+            this.$message.error(this.$t('Two passwords do not match'))
+            return
+          }
+          if (this.userInfo.password.length < 5) {
+            this.$message.error(this.$t('Password length should be no shorter than 5'))
+            return
+          }
+          const res = await this.$request.post(`/me/change-password`, {
+            password: this.userInfo.password
+          })
+          if (!res.data.error) {
+            this.$message.success(this.$t('Changed password successfully'))
+          }
+        })
      }
    }
  }
 </script>

 <style scoped>
-  .setting-form {
+  .setting-form,
+  .change-password-form {
    width: 600px;
  }

-  .setting-form .buttons {
+  .setting-form .buttons,
+  .change-password-form .buttons {
    text-align: right;
  }

-  .setting-form .icon {
+  .setting-form .icon,
+  .change-password-form .icon {
    top: calc(50% - 14px / 2);
    right: 14px;
    position: absolute;
    color: #DCDFE6;
  }

-  .setting-form >>> .el-form-item__label {
+  .setting-form >>> .el-form-item__label,
+  .change-password-form >>> .el-form-item__label {
    height: 40px;
  }

--- a/frontend/src/views/task/TaskDetail.vue
+++ b/frontend/src/views/task/TaskDetail.vue
@@ -213,24 +213,30 @@
        return ['pending', 'running'].includes(this.taskForm.status)
      }
    },
-    async created() {
+    async mounted() {
      await this.$store.dispatch('task/getTaskData', this.$route.params.id)

      this.isLogAutoFetch = !!this.isRunning
      this.isLogAutoScroll = !!this.isRunning

-      await this.$store.dispatch('task/getTaskResults', this.$route.params.id)
+      if (this.taskForm.type === 'spider') {
+        await this.$store.dispatch('task/getTaskResults', this.$route.params.id)
+      }

      await this.getTaskLog()
+
      this.handle = setInterval(async() => {
        if (this.isLogAutoFetch) {
          await this.$store.dispatch('task/getTaskData', this.$route.params.id)
-          await this.$store.dispatch('task/getTaskResults', this.$route.params.id)
+
          await this.getTaskLog()
+
+          if (this.taskForm.type === 'spider') {
+            await this.$store.dispatch('task/getTaskResults', this.$route.params.id)
+          }
        }
      }, 5000)
-    },
-    mounted() {
+
      if (!this.$utils.tour.isFinishedTour('task-detail')) {
        this.$utils.tour.startTour(this, 'task-detail')
      }
@@ -242,9 +248,6 @@
      onTabClick(tab) {
        this.$st.sendEv('任务详情', '切换标签', tab.name)
      },
-      onSpiderChange(id) {
-        this.$router.push(`/spiders/${id}`)
-      },
      onResultsPageChange(payload) {
        const { pageNum, pageSize } = payload
        this.resultsPageNum = pageNum
--- a/frontend/src/views/task/TaskList.vue
+++ b/frontend/src/views/task/TaskList.vue
@@ -14,6 +14,24 @@
      <div class="filter">
        <div class="left">
          <el-form class="filter-form" :model="filter" label-width="100px" label-position="right" inline>
+            <el-form-item :label="$t('Type')">
+              <el-button-group>
+                <el-button
+                  size="small"
+                  :type="filter.type === 'spider' ? 'primary' : ''"
+                  @click="onClickType('spider')"
+                >
+                  {{ $t('Spider Tasks') }}
+                </el-button>
+                <el-button
+                  size="small"
+                  :type="filter.type === 'system' ? 'primary' : ''"
+                  @click="onClickType('system')"
+                >
+                  {{ $t('System Tasks') }}
+                </el-button>
+              </el-button-group>
+            </el-form-item>
            <el-form-item prop="node_id" :label="$t('Node')">
              <el-select v-model="filter.node_id" size="small" :placeholder="$t('Node')" @change="onFilterChange">
                <el-option value="" :label="$t('All')" />
@@ -584,6 +602,11 @@
      onFilterChange() {
        this.$store.dispatch('task/getTaskList')
        this.$st.sendEv('任务列表', '筛选任务')
+      },
+      onClickType(type) {
+        this.$set(this.filter, 'type', type)
+        this.$store.dispatch('task/getTaskList')
+        this.$st.sendEv('任务列表', '选择类别', type)
      }
    }
  }