git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #496 from crawlab-team/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2020-02-01 18:17:14 +08:00
committed by GitHub
parent 06204ecbcf 83af0786e0
commit b6f73b5b77
29 changed files with 1069 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/conf/config.yml
View File

@@ -46,4 +46,4 @@ notification:
senderIdentity: ''
smtp:
user: ''
password: ''
password: ''

152
backend/main.go
View File

@@ -138,79 +138,99 @@ func main() {
}
authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
{
// 路由
// 节点
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
{
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
}
// 爬虫
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
{
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
}
// 可配置爬虫
authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
{
authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
}
// 任务
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
{
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
}
// 定时任务
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
{
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
}
// 用户
{
authGroup.GET("/users", routes.GetUserList) // 用户列表
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
authGroup.GET("/me", routes.GetMe) // 获取自己账户
authGroup.POST("/me", routes.PostMe) // 修改自己账户
}
// 系统
{
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
}
// 全局变量
{
authGroup.POST("/variable", routes.PostVariable) // 新增
authGroup.PUT("/variable/:id", routes.PutVariable) //修改
authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除
authGroup.GET("/variables", routes.GetVariableList) // 列表
}
// 统计数据
authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
// 用户
authGroup.GET("/users", routes.GetUserList) // 用户列表
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
authGroup.GET("/me", routes.GetMe) // 获取自己账户
authGroup.POST("/me", routes.PostMe) // 修改自己账户
// 系统
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
// 文件
authGroup.GET("/file", routes.GetFile) // 获取文件
}

97
backend/model/variable.go Normal file
View File

@@ -0,0 +1,97 @@
package model
import (
"crawlab/database"
"errors"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
"runtime/debug"
)
/**
全局变量
*/
type Variable struct {
Id bson.ObjectId `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
}
func (model *Variable) Save() error {
s, c := database.GetCol("variable")
defer s.Close()
if err := c.UpdateId(model.Id, model); err != nil {
log.Errorf("update variable error: %s", err.Error())
return err
}
return nil
}
func (model *Variable) Add() error {
s, c := database.GetCol("variable")
defer s.Close()
// key 去重
_, err := GetByKey(model.Key)
if err == nil {
return errors.New("key already exists")
}
model.Id = bson.NewObjectId()
if err := c.Insert(model); err != nil {
log.Errorf("add variable error: %s", err.Error())
debug.PrintStack()
return err
}
return nil
}
func (model *Variable) Delete() error {
s, c := database.GetCol("variable")
defer s.Close()
if err := c.RemoveId(model.Id); err != nil {
log.Errorf("remove variable error: %s", err.Error())
debug.PrintStack()
return err
}
return nil
}
func GetByKey(key string) (Variable, error) {
s, c := database.GetCol("variable")
defer s.Close()
var model Variable
if err := c.Find(bson.M{"key": key}).One(&model); err != nil {
log.Errorf("variable found error: %s, key: %s", err.Error(), key)
return model, err
}
return model, nil
}
func GetVariable(id bson.ObjectId) (Variable, error) {
s, c := database.GetCol("variable")
defer s.Close()
var model Variable
if err := c.FindId(id).One(&model); err != nil {
log.Errorf("variable found error: %s", err.Error())
return model, err
}
return model, nil
}
func GetVariableList() []Variable {
s, c := database.GetCol("variable")
defer s.Close()
var list []Variable
if err := c.Find(nil).All(&list); err != nil {
}
return list
}

62
backend/routes/variable.go Normal file
View File

@@ -0,0 +1,62 @@
package routes
import (
"crawlab/model"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
)
// 新增
func PostVariable(c *gin.Context) {
var variable model.Variable
if err := c.ShouldBindJSON(&variable); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
if err := variable.Add(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
// 修改
func PutVariable(c *gin.Context) {
var id = c.Param("id")
var variable model.Variable
if err := c.ShouldBindJSON(&variable); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
variable.Id = bson.ObjectIdHex(id)
if err := variable.Save(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
// 删除
func DeleteVariable(c *gin.Context) {
var idStr = c.Param("id")
var id = bson.ObjectIdHex(idStr)
variable, err := model.GetVariable(id)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
variable.Id = id
if err := variable.Delete(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
// 列表
func GetVariableList(c *gin.Context) {
list := model.GetVariableList()
HandleSuccessData(c, list)
}

11
backend/services/schedule.go
View File

@@ -200,6 +200,12 @@ func (s *Scheduler) Update() error {
return err
}
user, err := model.GetUserByUsername("admin")
if err != nil {
log.Errorf("get admin user error: %s", err.Error())
return err
}
// 遍历任务列表
for i := 0; i < len(sList); i++ {
// 单个任务
@@ -209,6 +215,11 @@ func (s *Scheduler) Update() error {
continue
}
// 兼容以前版本
if job.UserId.Hex() == "" {
job.UserId = user.Id
}
// 添加到定时任务
if err := s.AddJob(job); err != nil {
log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), job.Name, job.Cron)

16
backend/services/spider.go
View File

@@ -12,12 +12,11 @@ import (
"github.com/apex/log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
uuid "github.com/satori/go.uuid"
"github.com/satori/go.uuid"
"github.com/spf13/viper"
"os"
"path/filepath"
"runtime/debug"
"strings"
)
type SpiderFileData struct {
@@ -192,21 +191,14 @@ func PublishSpider(spider model.Spider) {
md5 := filepath.Join(path, spider_handler.Md5File)
if !utils.Exists(md5) {
log.Infof("md5 file not found: %s", md5)
spiderSync.RemoveSpiderFile()
spiderSync.Download()
spiderSync.CreateMd5File(gfFile.Md5)
spiderSync.RemoveDownCreate(gfFile.Md5)
return
}
// md5值不一样，则下载
md5Str := utils.ReadFileOneLine(md5)
// 去掉空格以及换行符
md5Str = strings.Replace(md5Str, " ", "", -1)
md5Str = strings.Replace(md5Str, "\n", "", -1)
md5Str := utils.GetSpiderMd5Str(md5)
if gfFile.Md5 != md5Str {
log.Infof("md5 is different, gf-md5:%s, file-md5:%s", gfFile.Md5, md5Str)
spiderSync.RemoveSpiderFile()
spiderSync.Download()
spiderSync.CreateMd5File(gfFile.Md5)
spiderSync.RemoveDownCreate(gfFile.Md5)
return
}
}

6
backend/services/spider_handler/spider.go
View File

@@ -38,6 +38,12 @@ func (s *SpiderSync) CreateMd5File(md5 string) {
}
}
func (s *SpiderSync) RemoveDownCreate(md5 string) {
s.RemoveSpiderFile()
s.Download()
s.CreateMd5File(md5)
}
// 获得下载锁的key
func (s *SpiderSync) GetLockDownloadKey(spiderId string) string {
node, _ := model.GetCurrentNode()

56
backend/services/task.go
View File

@@ -7,6 +7,7 @@ import (
"crawlab/lib/cron"
"crawlab/model"
"crawlab/services/notification"
"crawlab/services/spider_handler"
"crawlab/utils"
"encoding/json"
"errors"
@@ -144,7 +145,11 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exe
cmd.Env = append(cmd.Env, env.Name+"="+env.Value)
}
// TODO 全局环境变量
// 全局环境变量
variables := model.GetVariableList()
for _, variable := range variables {
cmd.Env = append(cmd.Env, variable.Key+"="+variable.Value)
}
return cmd
}
@@ -446,15 +451,9 @@ func ExecuteTask(id int) {
t.Status = constants.StatusRunning // 任务状态
t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长
// 判断爬虫文件是否存在
gfFile := model.GetGridFs(spider.FileId)
if gfFile == nil {
t.Error = "找不到爬虫文件，请重新上传"
t.Status = constants.StatusError
t.FinishTs = time.Now() // 结束时间
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
_ = t.Save()
// 文件检查
if err := SpiderFileCheck(t, spider); err != nil {
log.Errorf("spider file check error: %s", err.Error())
return
}
@@ -534,6 +533,30 @@ func ExecuteTask(id int) {
log.Infof(GetWorkerPrefix(id) + "任务(ID:" + t.Id + ")" + "执行完毕. 消耗时间:" + durationStr + "秒")
}
func SpiderFileCheck(t model.Task, spider model.Spider) error {
// 判断爬虫文件是否存在
gfFile := model.GetGridFs(spider.FileId)
if gfFile == nil {
t.Error = "找不到爬虫文件，请重新上传"
t.Status = constants.StatusError
t.FinishTs = time.Now() // 结束时间
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
_ = t.Save()
return errors.New(t.Error)
}
// 判断md5值是否一致
path := filepath.Join(viper.GetString("spider.path"), spider.Name)
md5File := filepath.Join(path, spider_handler.Md5File)
md5 := utils.GetSpiderMd5Str(md5File)
if gfFile.Md5 != md5 {
spiderSync := spider_handler.SpiderSync{Spider: spider}
spiderSync.RemoveDownCreate(gfFile.Md5)
}
return nil
}
func GetTaskLog(id string) (logStr string, err error) {
task, err := model.GetTask(id)
@@ -676,19 +699,6 @@ func AddTask(t model.Task) error {
return nil
}
func HandleTaskError(t model.Task, err error) {
log.Error("handle task error:" + err.Error())
t.Status = constants.StatusError
t.Error = err.Error()
t.FinishTs = time.Now()
if err := t.Save(); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
debug.PrintStack()
}
func GetTaskEmailMarkdownContent(t model.Task, s model.Spider) string {
n, _ := model.GetNode(t.NodeId)
errMsg := ""

7
backend/utils/file.go
View File

@@ -33,7 +33,14 @@ func ReadFileOneLine(fileName string) string {
return ""
}
return line
}
func GetSpiderMd5Str(file string) string {
md5Str := ReadFileOneLine(file)
// 去掉空格以及换行符
md5Str = strings.Replace(md5Str, " ", "", -1)
md5Str = strings.Replace(md5Str, "\n", "", -1)
return md5Str
}
// 创建文件

2
frontend/.env.development
View File

@@ -1,3 +1,3 @@
NODE_ENV='development'
VUE_APP_BASE_URL=http://localhost:8000
VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn
VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn

2
frontend/src/components/Config/ConfigList.vue
View File

@@ -477,6 +477,7 @@ export default {
} else if (currentStep === 12) {
this.activeTab = 'settings'
}
this.$utils.tour.prevStep('spider-detail-confg', currentStep)
},
onNextStep: (currentStep) => {
if (currentStep === 9) {
@@ -486,6 +487,7 @@ export default {
} else if (currentStep === 11) {
this.activeTab = 'spiderfile'
}
this.$utils.tour.nextStep('spider-detail-config', currentStep)
}
}
}

4
frontend/src/components/Cron/index.vue
View File

@@ -1,5 +1,5 @@
<style lang="scss" scoped>
#changeContab {
#change-crontab {
.language {
position: absolute;
right: 25px;
@@ -40,7 +40,7 @@
}
</style>
<template>
<div id="changeContab">
<div id="change-crontab">
<!-- <el-button class="language" type="text" @click="i18n=(i18n==='en'?'cn':'en')">{{i18n}}</el-button>-->
<el-tabs type="border-card">
<el-tab-pane>

13
frontend/src/components/File/FileList.vue
View File

@@ -203,18 +203,7 @@ export default {
dirDialogVisible: false,
fileDialogVisible: false,
nodeExpandedDict: {},
isShowDeleteNav: false,
tourSteps: [
{
target: '.add-btn',
content: this.$t('You can add a file or directory')
}
],
tourCallbacks: {
onStop: () => {
this.$utils.tour.finishTour('spider-detail-file-list')
}
}
isShowDeleteNav: false
}
},
computed: {

19
frontend/src/components/Node/NodeInstallation.vue
View File

@@ -1,8 +1,11 @@
<template>
<div class="node-installation">
<el-form inline>
<el-form class="search-form" inline>
<el-form-item>
<el-autocomplete size="small" clearable @clear="onSearch"
<el-autocomplete
class="search-box"
size="small"
clearable
v-if="activeLang.executable_name === 'python'"
v-model="depName"
style="width: 240px"
@@ -10,7 +13,8 @@
:fetchSuggestions="fetchAllDepList"
:minlength="2"
@select="onSearch"
></el-autocomplete>
@clear="onSearch"
/>
<el-input
v-else
v-model="depName"
@@ -20,9 +24,9 @@
</el-form-item>
<el-form-item>
<el-button size="small"
icon="el-icon-search"
type="success"
@click="onSearch"
icon="el-icon-search"
type="success"
@click="onSearch"
>
{{$t('Search')}}
</el-button>
@@ -116,7 +120,7 @@ export default {
depName: '',
depList: [],
loading: false,
isShowInstalled: false,
isShowInstalled: true,
installedDepList: [],
depLoadingDict: {},
isLoadingInstallLang: false
@@ -289,6 +293,7 @@ export default {
const res = await this.$request.get(`/nodes/${id}/langs`)
this.langList = res.data.data
this.activeTab = this.langList[0].executable_name || ''
await this.getInstalledDepList()
}
}
</script>

8
frontend/src/components/Overview/NodeOverview.vue
View File

@@ -2,14 +2,16 @@
<el-row>
<el-col :span="12">
<!--last tasks-->
<el-row>
<el-row class="latest-tasks-wrapper">
<task-table-view :title="$t('Latest Tasks')"/>
</el-row>
</el-col>
<el-col :span="12">
<!--basic info-->
<node-info-view/>
<el-row class="node-info-view-wrapper">
<!--basic info-->
<node-info-view/>
</el-row>
</el-col>
</el-row>
</template>

34
frontend/src/components/Overview/TaskOverview.vue
View File

@@ -1,7 +1,7 @@
<template>
<el-row>
<el-col :span="12" style="padding-right: 20px;">
<el-row>
<el-row class="task-info-overview-wrapper wrapper">
<h4 class="title">{{$t('Task Info')}}</h4>
<task-info-view/>
</el-row>
@@ -9,12 +9,16 @@
</el-col>
<el-col :span="12">
<el-row>
<h4 class="title spider-title" @click="onClickSpiderTitle">{{$t('Spider Info')}}</h4>
<el-row class="task-info-spider-wrapper wrapper">
<h4 class="title spider-title" @click="onClickSpiderTitle">
<i class="fa fa-search" style="margin-right: 5px"></i>
{{$t('Spider Info')}}</h4>
<spider-info-view :is-view="true"/>
</el-row>
<el-row>
<h4 class="title node-title" @click="onClickNodeTitle">{{$t('Node Info')}}</h4>
<el-row class="task-info-node-wrapper wrapper">
<h4 class="title node-title" @click="onClickNodeTitle">
<i class="fa fa-search" style="margin-right: 5px"></i>
{{$t('Node Info')}}</h4>
<node-info-view :is-view="true"/>
</el-row>
</el-col>
@@ -67,6 +71,12 @@ export default {
<style scoped>
.title {
margin: 10px 0 3px 0;
text-align: center;
display: inline-block;
}
.wrapper {
text-align: center;
}
.spider-form {
@@ -79,13 +89,17 @@ export default {
text-align: right;
}
.title {
text-align: center;
}
.node-title,
.spider-title {
text-decoration: underline;
cursor: pointer;
}
.node-title:hover,
.spider-title:hover {
text-decoration: underline;
}
.title > i {
color: grey;
}
</style>

38
frontend/src/i18n/zh.js
View File

@@ -365,6 +365,12 @@ export default {
'Never': '从不',
'DingTalk Robot Webhook': '钉钉机器人 Webhook',
'Wechat Robot Webhook': '微信机器人 Webhook',
'Password Settings': '密码设置',
'Notify Settings': '通知设置',
'Global Variable': '全局变量',
'Add Global Variable': '新增全局变量',
'Are you sure to delete this global variable': '确定删除该全局变量？',
'Key': '设置',
// 其他
tagsView: {
@@ -411,7 +417,7 @@ docker run -d --restart always --name crawlab_worker \\
'Previous': '上一步',
'Next': '下一步',
'Finish': '结束',
'Click to add a new spider': '点击并添加爬虫',
'Click to add a new spider.<br><br>You can also add a <strong>Customized Spider</strong> through <a href="https://docs.crawlab.cn/Usage/SDK/CLI.html" target="_blank" style="color: #409EFF">CLI Tool</a>.': '点击并添加爬虫<br><br>您也可以通过 <a href="https://docs.crawlab.cn/Usage/SDK/CLI.html" target="_blank" style="color: #409EFF">CLI 工具</a> 添加<strong>自定义爬虫</strong>',
'You can view your created spiders here.<br>Click a table row to view <strong>spider details</strong>.': '您可以查看创建的爬虫<br>点击行来查看<strong>爬虫详情</strong>',
'View a list of <strong>Configurable Spiders</strong>': '查看<strong>可配置爬虫</strong>列表',
'View a list of <strong>Customized Spiders</strong>': '查看<strong>自定义爬虫</strong>列表',
@@ -447,6 +453,36 @@ docker run -d --restart always --name crawlab_worker \\
'You can view the<br> visualization of the stage<br> workflow.': '您可以查看阶段工作流的<br>可视化界面',
'You can add the settings here, which will be loaded in the Scrapy\'s <code>settings.py</code> file.<br><br>JSON and Array data are supported.': '您可以在这里添加设置，它们会在 Scrapy 中的 <code>settings.py</code> 中被加载<br><br>JSON 和数组都支持',
'You can edit the <code>Spiderfile</code> here.<br><br>For more information, please refer to the <a href="https://docs.crawlab.cn/Usage/Spider/ConfigurableSpider.html" target="_blank" style="color: #409EFF">Documentation (Chinese)</a>.': '您可以在这里编辑 <code>Spiderfile</code><br><br>更多信息, 请参考 <a href="https://docs.crawlab.cn/Usage/Spider/ConfigurableSpider.html" target="_blank" style="color: #409EFF">文档</a>.',
'You can filter tasks from this area.': '您可以在这个区域筛选任务',
'This is a list of spider tasks executed sorted in a time descending order.': '这是执行过的爬虫任务的列表，按时间降序排列',
'Click the row to or the view button to view the task detail.': '点击行或查看按钮来查看任务详情',
'Tick and select the tasks you would like to delete in batches.': '勾选您想批量删除的任务',
'Click this button to delete selected tasks.': '点击并删除勾选的任务',
'This is the info of the task detail.': '这是任务详情信息',
'This is the spider info of the task.': '这是任务的爬虫信息',
'You can click to view the spider detail for the task.': '您可以点击查看该任务的爬虫详情',
'This is the node info of the task.': '这是任务的节点信息',
'You can click to view the node detail for the task.': '您可以点击查看该任务的节点详情',
'Here you can view the log<br> details for the task. The<br> log is automatically updated.': '这里您可以查看该任务<br>的日志详情，日志是<br>自动更新的',
'Here you can view the results scraped by the spider.<br><br><strong>Note:</strong> If you find your results here are empty, please refer to the <a href="https://docs.crawlab.cn/Integration/" target="_blank" style="color: #409EFF">Documentation (Chinese)</a> about how to integrate your spider into Crawlab.': '这里您可以查看爬虫抓取下来的结果<br><br><strong>注意:</strong> 如果这里结果是空的，请参考 <a href="https://docs.crawlab.cn/Integration/" target="_blank" style="color: #409EFF">相关文档</a> 来集成您的爬虫到 Crawlab',
'You can download your results as a CSV file by clicking this button.': '您可以点击下载结果为 CSV 文件',
'Switch between different nodes.': '在节点间切换',
'You can view the latest executed spider tasks.': '您可以查看最近执行过的爬虫任务',
'This is the detailed node info.': '这是节点详情',
'Here you can install<br> dependencies and modules<br> that are required<br> in your spiders.': '这里您可以安装您爬虫中<br>需要的依赖或模块',
'You can search dependencies in the search box and install them by clicking the "Install" button below.': '您可以在搜索框中搜索依赖并点击下面的"安装"按钮来进行安装',
'You should fill the form before adding the new schedule.': '在添加新定时任务前，您需要填写这个表单',
'The name of the schedule': '定时任务名称',
'The type of how to run the task.<br><br>Please refer to the <a href="https://docs.crawlab.cn/Usage/Spider/Run.html" target="_blank" style="color: #409EFF">Documentation (Chinese)</a> for detailed explanation for the options.<br><br>Let\'s select <strong>Selected Nodes</strong> for example.': '表示以哪种方式运行任务，<br><br>请参考 <a href="https://docs.crawlab.cn/Usage/Spider/Run.html" target="_blank" style="color: #409EFF">文档</a> 参考选项解释<br><br>让我们选择 <strong>指定节点</strong> 这个选项',
'The spider to run': '运行的爬虫',
'<strong>Cron</strong> expression for the schedule.<br><br>If you are not sure what a cron expression is, please refer to this <a href="https://baike.baidu.com/item/crontab/8819388" target="_blank" style="color: #409EFF">Article</a>.': '定时任务的 <strong>Cron</strong> 表达式<br><br>如果您不清楚什么是 Cron 表达式，请参考这篇 <a href="https://baike.baidu.com/item/crontab/8819388" target="_blank" style="color: #409EFF">文章（英文）</a>.',
'You can select the correct options in the cron config box to configure the cron expression.': '您可以在 Cron 配置栏里选择正确的选项来配置 Cron 表达式',
'The parameters which will be passed into the spider program.': '将被传入爬虫程序里的参数',
'The description for the schedule': '定时任务的描述',
'Once you have filled all fields, click this button to submit.': '当您填完所有字段，请点击这个按钮来提交定时任务',
'Here you can set your password.': '这里您可以设置您的密码',
'In this tab you can configure your notification settings.': '在这个标签中，您可以配置您的消息通知配置',
'Here you can add/edit/delete global environment variables which will be passed into your spider programs.': '这里您可以添加/修改/删除全局环境变量，它们会被传入爬虫程序中',
// 其他
'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'

2
frontend/src/store/index.js
View File

@@ -15,6 +15,7 @@ import site from './modules/site'
import stats from './modules/stats'
import setting from './modules/setting'
import version from './modules/version'
import tour from './modules/tour'
import getters from './getters'
Vue.use(Vuex)
@@ -35,6 +36,7 @@ const store = new Vuex.Store({
site,
setting,
version,
tour,
// 统计
stats
},

34
frontend/src/store/modules/tour.js Normal file
View File

@@ -0,0 +1,34 @@
const state = {
tourFinishSteps: {
'spider-list': 3,
'spider-list-add': 8,
'spider-detail': 9,
'spider-detail-config': 12,
'task-list': 4,
'task-detail': 7,
'node-detail': 4,
'schedule-list': 1,
'schedule-list-add': 8,
'setting': 2
},
tourSteps: {}
}
const getters = {}
const mutations = {
SET_TOUR_STEP: (state, payload) => {
const { tourName, step } = payload
state.tourSteps[tourName] = step
}
}
const actions = {}
export default {
namespaced: true,
state,
getters,
mutations,
actions
}

22
frontend/src/store/modules/user.js
View File

@@ -9,6 +9,8 @@ const user = {
avatar: '',
roles: [],
userList: [],
globalVariableList: [],
globalVariableForm: {},
userForm: {},
userInfo: undefined,
adminPaths: [
@@ -61,6 +63,9 @@ const user = {
},
SET_TOTAL_COUNT: (state, value) => {
state.totalCount = value
},
SET_GLOBAL_VARIABLE_LIST: (state, value) => {
state.globalVariableList = value
}
},
@@ -148,6 +153,23 @@ const user = {
// 添加用户
addUser ({ dispatch, commit, state }) {
return request.put('/users', state.userForm)
},
// 新增全局变量
addGlobalVariable ({ commit, state }) {
return request.post(`/variable`, state.globalVariableForm)
.then(() => {
state.globalVariableForm = {}
})
},
// 获取全局变量列表
getGlobalVariable ({ commit, state }) {
request.get('/variables').then((response) => {
commit('SET_GLOBAL_VARIABLE_LIST', response.data.data)
})
},
// 删除全局变量
deleteGlobalVariable ({ commit, state }, id) {
return request.delete(`/variable/${id}`)
}
}
}

25
frontend/src/utils/tour.js
View File

@@ -1,4 +1,6 @@
import i18n from '../i18n'
import store from '../store'
import stats from './stats'
export default {
isFinishedTour: (tourName) => {
@@ -26,6 +28,29 @@ export default {
}
data[tourName] = 1
localStorage.setItem('tour', JSON.stringify(data))
// 发送统计数据
const finalStep = store.state.tour.tourFinishSteps[tourName]
const currentStep = store.state.tour.tourSteps[tourName]
if (currentStep === finalStep) {
stats.sendEv('教程', '完成', tourName)
} else {
stats.sendEv('教程', '跳过', tourName)
}
},
nextStep: (tourName, currentStep) => {
store.commit('tour/SET_TOUR_STEP', {
tourName,
step: currentStep + 1
})
stats.sendEv('教程', '下一步', tourName)
},
prevStep: (tourName, currentStep) => {
store.commit('tour/SET_TOUR_STEP', {
tourName,
step: currentStep - 1
})
stats.sendEv('教程', '上一步', tourName)
},
getOptions: (isShowHighlight) => {
return {

64
frontend/src/views/node/NodeDetail.vue
View File

@@ -1,5 +1,14 @@
<template>
<div class="app-container">
<!--tour-->
<v-tour
name="node-detail"
:steps="tourSteps"
:callbacks="tourCallbacks"
:options="$utils.tour.getOptions(true)"
/>
<!--./tour-->
<!--selector-->
<div class="selector">
<label class="label">{{$t('Node')}}: </label>
@@ -38,7 +47,54 @@ export default {
},
data () {
return {
activeTabName: 'overview'
activeTabName: 'overview',
tourSteps: [
// overview
{
target: '.selector',
content: this.$t('Switch between different nodes.')
},
{
target: '.latest-tasks-wrapper',
content: this.$t('You can view the latest executed spider tasks.'),
params: {
placement: 'right'
}
},
{
target: '.node-info-view-wrapper',
content: this.$t('This is the detailed node info.'),
params: {
placement: 'left'
}
},
// installation
{
target: '#tab-installation',
content: this.$t('Here you can install<br> dependencies and modules<br> that are required<br> in your spiders.')
},
{
target: '.search-box',
content: this.$t('You can search dependencies in the search box and install them by clicking the "Install" button below.')
}
],
tourCallbacks: {
onStop: () => {
this.$utils.tour.finishTour('node-detail')
},
onPreviousStep: (currentStep) => {
if (currentStep === 3) {
this.activeTabName = 'overview'
}
this.$utils.tour.prevStep('node-detail', currentStep)
},
onNextStep: (currentStep) => {
if (currentStep === 2) {
this.activeTabName = 'installation'
}
this.$utils.tour.nextStep('node-detail', currentStep)
}
}
}
},
computed: {
@@ -65,6 +121,12 @@ export default {
// get node task list
this.$store.dispatch('node/getTaskList', this.$route.params.id)
},
mounted () {
if (!this.$utils.tour.isFinishedTour('node-detail')) {
this.$tours['node-detail'].start()
this.$st.sendEv('教程', '开始', 'node-detail')
}
}
}
</script>

95
frontend/src/views/result/ResultDetail.vue
View File

@@ -1,95 +0,0 @@
<template>
<div class="app-container">
<!--selector-->
<div class="selector">
<label class="label">Spider: </label>
<el-select v-model="spiderForm._id" @change="onSpiderChange">
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
</div>
<!--tabs-->
<el-tabs v-model="activeTabName" @tab-click="onTabClick" type="card">
<el-tab-pane label="Overview" name="overview">
<spider-overview/>
</el-tab-pane>
<el-tab-pane label="Files" name="files">
<file-list/>
</el-tab-pane>
</el-tabs>
</div>
</template>
<script>
import {
mapState
} from 'vuex'
import FileList from '../../components/File/FileList'
import SpiderOverview from '../../components/Overview/SpiderOverview'
export default {
name: 'ResultDetail',
components: {
FileList,
SpiderOverview
},
data () {
return {
activeTabName: 'overview'
}
},
computed: {
...mapState('spider', [
'spiderList',
'spiderForm'
]),
...mapState('file', [
'currentPath'
]),
...mapState('deploy', [
'deployList'
])
},
methods: {
onTabClick () {
},
onSpiderChange (id) {
this.$router.push(`/spiders/${id}`)
}
},
created () {
// get the list of the spiders
// this.$store.dispatch('spider/getSpiderList')
// get spider basic info
this.$store.dispatch('spider/getSpiderData', this.$route.params.id)
.then(() => {
// get spider file info
this.$store.dispatch('file/getFileList', this.spiderForm.src)
})
// get spider deploys
this.$store.dispatch('spider/getDeployList', this.$route.params.id)
// get spider tasks
this.$store.dispatch('spider/getTaskList', this.$route.params.id)
}
}
</script>
<style scoped>
.selector {
display: flex;
align-items: center;
position: absolute;
right: 20px;
/*float: right;*/
z-index: 999;
margin-top: -7px;
}
.selector .el-select {
padding-left: 10px;
}
</style>

189
frontend/src/views/schedule/ScheduleList.vue
View File

@@ -1,28 +1,44 @@
<template>
<div class="app-container">
<!--tour-->
<v-tour
name="schedule-list"
:steps="tourSteps"
:callbacks="tourCallbacks"
:options="$utils.tour.getOptions(true)"
/>
<v-tour
name="schedule-list-add"
:steps="tourAddSteps"
:callbacks="tourAddCallbacks"
:options="$utils.tour.getOptions(true)"
/>
<!--./tour-->
<!--add popup-->
<el-dialog
:title="$t(dialogTitle)"
:visible.sync="dialogVisible"
width="60%"
width="640px"
:before-close="onDialogClose">
<el-form label-width="180px"
class="add-form"
:model="scheduleForm"
:inline-message="true"
ref="scheduleForm"
label-position="right">
<el-form-item :label="$t('Schedule Name')" prop="name" required>
<el-input v-model="scheduleForm.name" :placeholder="$t('Schedule Name')"></el-input>
<el-input id="schedule-name" v-model="scheduleForm.name" :placeholder="$t('Schedule Name')"></el-input>
</el-form-item>
<el-form-item :label="$t('Run Type')" prop="run_type" required>
<el-select v-model="scheduleForm.run_type" :placeholder="$t('Run Type')">
<el-select id="run-type" v-model="scheduleForm.run_type" :placeholder="$t('Run Type')">
<el-option value="all-nodes" :label="$t('All Nodes')"/>
<el-option value="selected-nodes" :label="$t('Selected Nodes')"/>
<el-option value="random" :label="$t('Random')"/>
</el-select>
</el-form-item>
<el-form-item v-if="scheduleForm.run_type === 'selected-nodes'" :label="$t('Nodes')" prop="node_ids" required>
<el-select v-model="scheduleForm.node_ids" :placeholder="$t('Nodes')" multiple filterable>
<el-select id="node-ids" v-model="scheduleForm.node_ids" :placeholder="$t('Nodes')" multiple filterable>
<el-option
v-for="op in nodeList"
:key="op._id"
@@ -34,6 +50,7 @@
</el-form-item>
<el-form-item v-if="!isDisabledSpiderSchedule" :label="$t('Spider')" prop="spider_id" required>
<el-select
id="spider-id"
v-model="scheduleForm.spider_id"
:placeholder="$t('Spider')"
filterable
@@ -67,14 +84,18 @@
</el-select>
</el-form-item>
<el-form-item :label="$t('Cron')" prop="cron" required>
<el-popover>
<el-popover v-model="isShowCron" trigger="manual">
<template>
<vue-cron-linux :data="scheduleForm.cron" :i18n="lang" @change="onCronChange"/>
</template>
<template slot="reference">
<el-input
id="cron"
ref="cron"
v-model="scheduleForm.cron"
:placeholder="`${$t('[minute] [hour] [day] [month] [day of week]')}`"
@focus="isShowCron = true"
@blur="isShowCron = false"
>
</el-input>
</template>
@@ -82,24 +103,29 @@
<!--<el-button size="small" style="width:100px" type="primary" @click="onShowCronDialog">{{$t('schedules.add_cron')}}</el-button>-->
</el-form-item>
<el-form-item :label="$t('Execute Command')" prop="params">
<el-input v-model="spider.cmd"
:placeholder="$t('Execute Command')"
disabled>
</el-input>
<el-input
id="cmd"
v-model="spider.cmd"
:placeholder="$t('Execute Command')"
disabled
/>
</el-form-item>
<el-form-item :label="$t('Parameters')" prop="param">
<el-input v-model="scheduleForm.param"
:placeholder="$t('Parameters')"></el-input>
<el-input
id="param"
v-model="scheduleForm.param"
:placeholder="$t('Parameters')"
/>
</el-form-item>
<el-form-item :label="$t('Schedule Description')" prop="description">
<el-input v-model="scheduleForm.description" type="textarea"
<el-input id="schedule-description" v-model="scheduleForm.description" type="textarea"
:placeholder="$t('Schedule Description')"></el-input>
</el-form-item>
</el-form>
<!--取消、保存-->
<span slot="footer" class="dialog-footer">
<el-button size="small" @click="onCancel">{{$t('Cancel')}}</el-button>
<el-button size="small" type="primary" @click="onAddSubmit">{{$t('Submit')}}</el-button>
<el-button id="btn-submit" size="small" type="primary" @click="onAddSubmit">{{$t('Submit')}}</el-button>
</span>
</el-dialog>
@@ -114,7 +140,7 @@
<div class="right">
<el-button size="small" type="primary"
icon="el-icon-plus"
class="refresh"
class="btn-add"
@click="onAdd">
{{$t('Add Schedule')}}
</el-button>
@@ -233,7 +259,125 @@ export default {
showCron: false,
expression: '',
spiderList: [],
nodeList: []
nodeList: [],
isShowCron: false,
// tutorial
tourSteps: [
{
target: '.table',
content: this.$t('This is a list of schedules (cron jobs) to periodically run spider tasks. You can add/modify/edit your schedules here.<br><br>For more information, please refer to the <a href="https://docs.crawlab.cn/Usage/Schedule/" target="_blank" style="color: #409EFF">Documentation (Chinese)</a> for detail.')
},
{
target: '.btn-add',
content: this.$t('You can add a new schedule by clicking this button.')
}
],
tourCallbacks: {
onStop: () => {
this.$utils.tour.finishTour('schedule-list')
},
onPreviousStep: (currentStep) => {
if (currentStep === 2) {
this.dialogVisible = false
}
this.$utils.tour.prevStep('schedule-list', currentStep)
},
onNextStep: (currentStep) => {
if (currentStep === 1) {
this.isEdit = false
this.dialogVisible = true
this.$store.commit('schedule/SET_SCHEDULE_FORM', { node_ids: [] })
}
this.$utils.tour.nextStep('schedule-list', currentStep)
}
},
tourAddSteps: [
{
target: '.add-form',
content: this.$t('You should fill the form before adding the new schedule.'),
params: {
placement: 'right'
}
},
{
target: '#schedule-name',
content: this.$t('The name of the schedule'),
params: {
placement: 'right'
}
},
{
target: '#run-type',
content: this.$t('The type of how to run the task.<br><br>Please refer to the <a href="https://docs.crawlab.cn/Usage/Spider/Run.html" target="_blank" style="color: #409EFF">Documentation (Chinese)</a> for detailed explanation for the options.<br><br>Let\'s select <strong>Selected Nodes</strong> for example.'),
params: {
placement: 'right'
}
},
{
target: '#spider-id',
content: this.$t('The spider to run'),
params: {
placement: 'right'
}
},
{
target: '#cron',
content: this.$t('<strong>Cron</strong> expression for the schedule.<br><br>If you are not sure what a cron expression is, please refer to this <a href="https://baike.baidu.com/item/crontab/8819388" target="_blank" style="color: #409EFF">Article</a>.'),
params: {
placement: 'right'
}
},
{
target: '#change-crontab',
content: this.$t('You can select the correct options in the cron config box to configure the cron expression.'),
params: {
placement: 'top'
}
},
{
target: '#param',
content: this.$t('The parameters which will be passed into the spider program.'),
params: {
placement: 'right'
}
},
{
target: '#schedule-description',
content: this.$t('The description for the schedule'),
params: {
placement: 'right'
}
},
{
target: '#btn-submit',
content: this.$t('Once you have filled all fields, click this button to submit.'),
params: {
placement: 'right'
}
}
],
tourAddCallbacks: {
onStop: () => {
this.$utils.tour.finishTour('schedule-list-add')
},
onPreviousStep: (currentStep) => {
if (currentStep === 4) {
this.isShowCron = false
} else if (currentStep === 6) {
this.isShowCron = true
}
this.$utils.tour.prevStep('schedule-list-add', currentStep)
},
onNextStep: (currentStep) => {
if (currentStep === 3) {
this.isShowCron = true
} else if (currentStep === 5) {
this.isShowCron = false
}
this.$utils.tour.nextStep('schedule-list-add', currentStep)
}
}
}
},
computed: {
@@ -274,6 +418,13 @@ export default {
this.dialogVisible = true
this.$store.commit('schedule/SET_SCHEDULE_FORM', { node_ids: [] })
this.$st.sendEv('定时任务', '添加定时任务')
if (!this.$utils.tour.isFinishedTour('schedule-list-add')) {
setTimeout(() => {
this.$tours['schedule-list-add'].start()
this.$st.sendEv('教程', '开始', 'schedule-list-add')
}, 500)
}
},
onAddSubmit () {
this.$refs.scheduleForm.validate(res => {
@@ -377,6 +528,14 @@ export default {
.then(response => {
this.spiderList = response.data.data.list || []
})
},
mounted () {
if (!this.isDisabledSpiderSchedule) {
if (!this.$utils.tour.isFinishedTour('schedule-list')) {
this.$tours['schedule-list'].start()
this.$st.sendEv('教程', '开始', 'schedule-list')
}
}
}
}
</script>

241
frontend/src/views/setting/Setting.vue
View File

@@ -1,54 +1,112 @@
<template>
<div class="app-container">
<el-form :model="userInfo" class="setting-form" ref="setting-form" label-width="200px" :rules="rules"
inline-message>
<el-form-item prop="username" :label="$t('Username')">
<el-input v-model="userInfo.username" disabled></el-input>
</el-form-item>
<el-form-item prop="password" :label="$t('Password')">
<el-input v-model="userInfo.password" type="password" :placeholder="$t('Password')"></el-input>
</el-form-item>
<div style="border-bottom: 1px solid #DCDFE6"></div>
<el-form-item :label="$t('Notification Trigger Timing')">
<el-radio-group v-model="userInfo.setting.notification_trigger">
<el-radio label="notification_trigger_on_task_end">
{{$t('On Task End')}}
</el-radio>
<el-radio label="notification_trigger_on_task_error">
{{$t('On Task Error')}}
</el-radio>
<el-radio label="notification_trigger_never">
{{$t('Never')}}
</el-radio>
</el-radio-group>
</el-form-item>
<el-form-item prop="enabledNotifications" :label="$t('消息通知方式')">
<el-checkbox-group v-model="userInfo.setting.enabled_notifications">
<el-checkbox label="notification_type_mail">{{$t('邮件')}}</el-checkbox>
<el-checkbox label="notification_type_ding_talk">{{$t('钉钉')}}</el-checkbox>
<el-checkbox label="notification_type_wechat">{{$t('企业微信')}}</el-checkbox>
</el-checkbox-group>
</el-form-item>
<el-form-item prop="email" :label="$t('Email')">
<el-input v-model="userInfo.email" :placeholder="$t('Email')"></el-input>
</el-form-item>
<el-form-item prop="setting.ding_talk_robot_webhook" :label="$t('DingTalk Robot Webhook')">
<el-input v-model="userInfo.setting.ding_talk_robot_webhook"
:placeholder="$t('DingTalk Robot Webhook')"></el-input>
</el-form-item>
<el-form-item prop="setting.wechat_robot_webhook" :label="$t('Wechat Robot Webhook')">
<el-input v-model="userInfo.setting.wechat_robot_webhook" :placeholder="$t('Wechat Robot Webhook')"></el-input>
</el-form-item>
<el-form-item>
<div class="buttons">
<el-button type="success" @click="saveUserInfo">{{$t('Save')}}</el-button>
<!--tour-->
<v-tour
name="setting"
:steps="tourSteps"
:callbacks="tourCallbacks"
:options="$utils.tour.getOptions(true)"
/>
<!--./tour-->
<!-- 新增全局变量 -->
<el-dialog :title="$t('Add Global Variable')"
:visible.sync="addDialogVisible">
<el-form label-width="80px" ref="globalVariableForm">
<el-form-item :label="$t('Key')">
<el-input size="small" v-model="globalVariableForm.key"/>
</el-form-item>
<el-form-item :label="$t('Value')">
<el-input size="small" v-model="globalVariableForm.value"/>
</el-form-item>
<el-form-item :label="$t('Remark')">
<el-input size="small" v-model="globalVariableForm.remark"/>
</el-form-item>
<el-form-item>
<div style="text-align: right">
<el-button @click="addDialogVisible = false" type="danger" size="small">{{$t('Cancel')}}</el-button>
<el-button @click="addGlobalVariableHandle(false)" type="success" size="small">{{$t('Save')}}</el-button>
</div>
</el-form-item>
</el-form>
</el-dialog>
<el-tabs v-model="activeName" @tab-click="tabActiveHandle">
<el-tab-pane :label="$t('Password Settings')" name="password">
<el-form :model="userInfo" class="setting-form" ref="setting-form" label-width="200px" :rules="rules"
inline-message>
<el-form-item prop="username" :label="$t('Username')">
<el-input v-model="userInfo.username" disabled></el-input>
</el-form-item>
<el-form-item prop="password" :label="$t('Password')">
<el-input v-model="userInfo.password" type="password" :placeholder="$t('Password')"></el-input>
</el-form-item>
</el-form>
</el-tab-pane>
<el-tab-pane :label="$t('Notify Settings')" name="notify">
<el-form :model="userInfo" class="setting-form" ref="setting-form" label-width="200px" :rules="rules"
inline-message>
<el-form-item :label="$t('Notification Trigger Timing')">
<el-radio-group v-model="userInfo.setting.notification_trigger">
<el-radio label="notification_trigger_on_task_end">
{{$t('On Task End')}}
</el-radio>
<el-radio label="notification_trigger_on_task_error">
{{$t('On Task Error')}}
</el-radio>
<el-radio label="notification_trigger_never">
{{$t('Never')}}
</el-radio>
</el-radio-group>
</el-form-item>
<el-form-item prop="enabledNotifications" :label="$t('消息通知方式')">
<el-checkbox-group v-model="userInfo.setting.enabled_notifications">
<el-checkbox label="notification_type_mail">{{$t('邮件')}}</el-checkbox>
<el-checkbox label="notification_type_ding_talk">{{$t('钉钉')}}</el-checkbox>
<el-checkbox label="notification_type_wechat">{{$t('企业微信')}}</el-checkbox>
</el-checkbox-group>
</el-form-item>
<el-form-item prop="email" :label="$t('Email')">
<el-input v-model="userInfo.email" :placeholder="$t('Email')"></el-input>
</el-form-item>
<el-form-item prop="setting.ding_talk_robot_webhook" :label="$t('DingTalk Robot Webhook')">
<el-input v-model="userInfo.setting.ding_talk_robot_webhook"
:placeholder="$t('DingTalk Robot Webhook')"></el-input>
</el-form-item>
<el-form-item prop="setting.wechat_robot_webhook" :label="$t('Wechat Robot Webhook')">
<el-input v-model="userInfo.setting.wechat_robot_webhook"
:placeholder="$t('Wechat Robot Webhook')"></el-input>
</el-form-item>
</el-form>
</el-tab-pane>
<el-tab-pane :label="$t('Global Variable')" name="global-variable">
<div style="text-align: right;margin-bottom: 10px">
<el-button size="small" @click="addGlobalVariableHandle(true)"
icon="el-icon-plus"
type="primary">
{{$t('Add')}}
</el-button>
<el-button size="small" type="success" @click="saveUserInfo">{{$t('Save')}}</el-button>
</div>
</el-form-item>
</el-form>
<el-table :data="globalVariableList" border>
<el-table-column prop="key" :label="$t('Key')"/>
<el-table-column prop="value" :label="$t('Value')"/>
<el-table-column prop="remark" :label="$t('Remark')"/>
<el-table-column prop="" :label="$t('Action')" width="80">
<template slot-scope="scope">
<el-button @click="deleteGlobalVariableHandle(scope.row._id)" icon="el-icon-delete" type="danger"
size="mini"></el-button>
</template>
</el-table-column>
</el-table>
</el-tab-pane>
</el-tabs>
</div>
</template>
<script>
import { mapState } from 'vuex'
export default {
name: 'Setting',
data () {
@@ -92,16 +150,94 @@ export default {
'setting.ding_talk_robot_webhook': [{ trigger: 'blur', validator: validateDingTalkRobotWebhook }],
'setting.wechat_robot_webhook': [{ trigger: 'blur', validator: validateWechatRobotWebhook }]
},
isShowDingTalkAppSecret: false
isShowDingTalkAppSecret: false,
activeName: 'password',
addDialogVisible: false,
tourSteps: [
{
target: '#tab-password',
content: this.$t('Here you can set your password.'),
params: {
placement: 'right'
}
},
{
target: '#tab-notify',
content: this.$t('In this tab you can configure your notification settings.')
},
{
target: '#tab-global-variable',
content: this.$t('Here you can add/edit/delete global environment variables which will be passed into your spider programs.')
}
],
tourCallbacks: {
onStop: () => {
this.$utils.tour.finishTour('setting')
},
onPreviousStep: (currentStep) => {
if (currentStep === 1) {
this.activeName = 'password'
} else if (currentStep === 2) {
this.activeName = 'notify'
}
this.$utils.tour.prevStep('setting', currentStep)
},
onNextStep: (currentStep) => {
if (currentStep === 0) {
this.activeName = 'notify'
} else if (currentStep === 1) {
this.activeName = 'global-variable'
}
this.$utils.tour.nextStep('setting', currentStep)
}
}
}
},
computed: {
...mapState('user', [
'globalVariableList',
'globalVariableForm'
])
},
methods: {
deleteGlobalVariableHandle (id) {
this.$confirm(this.$t('Are you sure to delete this global variable'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel'),
type: 'warning'
}).then(() => {
this.$store.dispatch('user/deleteGlobalVariable', id).then(() => {
this.$store.dispatch('user/getGlobalVariable')
})
}).catch(() => {
})
},
addGlobalVariableHandle (isShow) {
if (isShow) {
this.addDialogVisible = true
return
}
this.$store.dispatch('user/addGlobalVariable')
.then(() => {
this.addDialogVisible = false
this.$st.sendEv('设置', '添加全局变量')
})
.then(() => {
this.$store.dispatch('user/getGlobalVariable')
})
},
getUserInfo () {
const data = localStorage.getItem('user_info')
if (!data) return {}
if (!data) {
return {}
}
this.userInfo = JSON.parse(data)
if (!this.userInfo.setting) this.userInfo.setting = {}
if (!this.userInfo.setting.enabled_notifications) this.userInfo.setting.enabled_notifications = []
if (!this.userInfo.setting) {
this.userInfo.setting = {}
}
if (!this.userInfo.setting.enabled_notifications) {
this.userInfo.setting.enabled_notifications = []
}
},
saveUserInfo () {
this.$refs['setting-form'].validate(async valid => {
@@ -114,11 +250,20 @@ export default {
}
})
this.$st.sendEv('设置', '保存')
},
tabActiveHandle () {
}
},
async created () {
await this.$store.dispatch('user/getInfo')
await this.$store.dispatch('user/getGlobalVariable')
this.getUserInfo()
},
mounted () {
if (!this.$utils.tour.isFinishedTour('setting')) {
this.$tours['setting'].start()
this.$st.sendEv('教程', '开始', 'setting')
}
}
}
</script>

4
frontend/src/views/spider/SpiderDetail.vue
View File

@@ -143,6 +143,7 @@ export default {
} else if (currentStep === 9) {
this.activeTabName = 'environment'
}
this.$utils.tour.prevStep('spider-detail', currentStep)
},
onNextStep: (currentStep) => {
if (currentStep === 4) {
@@ -152,6 +153,7 @@ export default {
} else if (currentStep === 8) {
this.activeTabName = 'schedules'
}
this.$utils.tour.nextStep('spider-detail', currentStep)
}
}
}
@@ -188,6 +190,7 @@ export default {
if (!this.$utils.tour.isFinishedTour('spider-detail-config')) {
setTimeout(() => {
this.$tours['spider-detail-config'].start()
this.$st.sendEv('教程', '开始', 'spider-detail-config')
}, 100)
}
}
@@ -222,6 +225,7 @@ export default {
mounted () {
if (!this.$utils.tour.isFinishedTour('spider-detail')) {
this.$tours['spider-detail'].start()
this.$st.sendEv('教程', '开始', 'spider-detail')
}
}
}

12
frontend/src/views/spider/SpiderList.vue
View File

@@ -380,12 +380,18 @@ export default {
},
{
target: '.btn.add',
content: this.$t('Click to add a new spider')
content: this.$t('Click to add a new spider.<br><br>You can also add a <strong>Customized Spider</strong> through <a href="https://docs.crawlab.cn/Usage/SDK/CLI.html" target="_blank" style="color: #409EFF">CLI Tool</a>.')
}
],
tourCallbacks: {
onStop: () => {
this.$utils.tour.finishTour('spider-list')
},
onPreviousStep: (currentStep) => {
this.$utils.tour.prevStep('spider-list', currentStep)
},
onNextStep: (currentStep) => {
this.$utils.tour.nextStep('spider-list', currentStep)
}
},
tourAddSteps: [
@@ -463,11 +469,13 @@ export default {
if (currentStep === 7) {
this.spiderType = 'customized'
}
this.$utils.tour.prevStep('spider-list-add', currentStep)
},
onNextStep: (currentStep) => {
if (currentStep === 6) {
this.spiderType = 'configurable'
}
this.$utils.tour.nextStep('spider-list-add', currentStep)
}
}
}
@@ -517,6 +525,7 @@ export default {
setTimeout(() => {
if (!this.$utils.tour.isFinishedTour('spider-list-add')) {
this.$tours['spider-list-add'].start()
this.$st.sendEv('教程', '开始', 'spider-list-add')
}
}, 300)
},
@@ -751,6 +760,7 @@ export default {
if (!this.$utils.tour.isFinishedTour('spider-list')) {
this.$tours['spider-list'].start()
this.$st.sendEv('教程', '开始', 'spider-list')
}
}
}

94
frontend/src/views/task/TaskDetail.vue
View File

@@ -1,5 +1,14 @@
<template>
<div class="app-container">
<!--tour-->
<v-tour
name="task-detail"
:steps="tourSteps"
:callbacks="tourCallbacks"
:options="$utils.tour.getOptions(true)"
/>
<!--./tour-->
<!--tabs-->
<el-tabs v-model="activeTabName" @tab-click="onTabClick" type="card">
<el-tab-pane :label="$t('Overview')" name="overview">
@@ -12,7 +21,7 @@
</el-tab-pane>
<el-tab-pane :label="$t('Results')" name="results">
<div class="button-group">
<el-button type="primary" icon="el-icon-download" @click="downloadCSV">
<el-button class="btn-download" type="primary" icon="el-icon-download" @click="downloadCSV">
{{$t('Download CSV')}}
</el-button>
</div>
@@ -48,7 +57,82 @@ export default {
return {
activeTabName: 'overview',
handle: undefined,
taskLog: ''
taskLog: '',
// tutorial
tourSteps: [
// overview
{
target: '.task-info-overview-wrapper',
content: this.$t('This is the info of the task detail.'),
params: {
placement: 'right'
}
},
{
target: '.task-info-spider-wrapper',
content: this.$t('This is the spider info of the task.'),
params: {
placement: 'left'
}
},
{
target: '.spider-title',
content: this.$t('You can click to view the spider detail for the task.'),
params: {
placement: 'left'
}
},
{
target: '.task-info-node-wrapper',
content: this.$t('This is the node info of the task.'),
params: {
placement: 'left'
}
},
{
target: '.node-title',
content: this.$t('You can click to view the node detail for the task.'),
params: {
placement: 'left'
}
},
// log
{
target: '#tab-log',
content: this.$t('Here you can view the log<br> details for the task. The<br> log is automatically updated.')
},
// results
{
target: '#tab-results',
content: this.$t('Here you can view the results scraped by the spider.<br><br><strong>Note:</strong> If you find your results here are empty, please refer to the <a href="https://docs.crawlab.cn/Integration/" target="_blank" style="color: #409EFF">Documentation (Chinese)</a> about how to integrate your spider into Crawlab.')
},
{
target: '.btn-download',
content: this.$t('You can download your results as a CSV file by clicking this button.')
}
],
tourCallbacks: {
onStop: () => {
this.$utils.tour.finishTour('task-detail')
},
onPreviousStep: (currentStep) => {
if (currentStep === 5) {
this.activeTabName = 'overview'
} else if (currentStep === 6) {
this.activeTabName = 'log'
}
this.$utils.tour.prevStep('task-detail', currentStep)
},
onNextStep: (currentStep) => {
if (currentStep === 4) {
this.activeTabName = 'log'
} else if (currentStep === 5) {
this.activeTabName = 'results'
}
this.$utils.tour.nextStep('task-detail', currentStep)
}
}
}
},
computed: {
@@ -116,6 +200,12 @@ export default {
this.getTaskLog()
}, 5000)
},
mounted () {
if (!this.$utils.tour.isFinishedTour('task-detail')) {
this.$tours['task-detail'].start()
this.$st.sendEv('教程', '开始', 'task-detail')
}
},
destroyed () {
clearInterval(this.handle)
}

61
frontend/src/views/task/TaskList.vue
View File

@@ -1,10 +1,19 @@
<template>
<div class="app-container">
<!--tour-->
<v-tour
name="task-list"
:steps="tourSteps"
:callbacks="tourCallbacks"
:options="$utils.tour.getOptions(true)"
/>
<!--./tour-->
<el-card style="border-radius: 0">
<!--filter-->
<div class="filter">
<div class="left">
<el-form :model="filter" label-width="100px" label-position="right" inline>
<el-form class="filter-form" :model="filter" label-width="100px" label-position="right" inline>
<el-form-item prop="node_id" :label="$t('Node')">
<el-select v-model="filter.node_id" size="small" :placeholder="$t('Node')" @change="onFilterChange">
<el-option value="" :label="$t('All')"/>
@@ -29,7 +38,7 @@
</el-form>
</div>
<div class="right">
<el-button @click="onRemoveMultipleTask" size="small" type="danger">
<el-button class="btn-delete" @click="onRemoveMultipleTask" size="small" type="danger">
删除任务
</el-button>
</div>
@@ -191,7 +200,48 @@ export default {
// { name: 'avg_num_results', label: 'Average Results Count per Second', width: '80' }
],
multipleSelection: []
multipleSelection: [],
// tutorial
tourSteps: [
{
target: '.filter-form',
content: this.$t('You can filter tasks from this area.')
},
{
target: '.table',
content: this.$t('This is a list of spider tasks executed sorted in a time descending order.')
},
{
target: '.table .el-table__body-wrapper tr:nth-child(1)',
content: this.$t('Click the row to or the view button to view the task detail.')
},
{
target: '.table tr td:nth-child(1)',
content: this.$t('Tick and select the tasks you would like to delete in batches.'),
params: {
placement: 'right'
}
},
{
target: '.btn-delete',
content: this.$t('Click this button to delete selected tasks.'),
params: {
placement: 'left'
}
}
],
tourCallbacks: {
onStop: () => {
this.$utils.tour.finishTour('task-list')
},
onPreviousStep: (currentStep) => {
this.$utils.tour.prevStep('task-list', currentStep)
},
onNextStep: (currentStep) => {
this.$utils.tour.nextStep('task-list', currentStep)
}
}
}
},
computed: {
@@ -350,6 +400,11 @@ export default {
this.handle = setInterval(() => {
this.$store.dispatch('task/getTaskList')
}, 5000)
if (!this.$utils.tour.isFinishedTour('task-list')) {
this.$tours['task-list'].start()
this.$st.sendEv('教程', '开始', 'task-list')
}
},
destroyed () {
clearInterval(this.handle)