diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index bc2b213f..bbb63eae 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,7 +1,10 @@ # 0.4.5 (unkown) ### 功能 / 优化 - **交互式教程**. 引导用户了解 Crawlab 的主要功能. -- **加入全局环境变量**. 可以设置全局环境变量，然后传入到所有爬虫程序中. +- **加入全局环境变量**. 可以设置全局环境变量，然后传入到所有爬虫程序中. [#177](https://github.com/crawlab-team/crawlab/issues/177) +- **项目**. 允许用户将爬虫关联到项目上. [#316](https://github.com/crawlab-team/crawlab/issues/316) +- **用户管理优化**. 限制管理用户的权限. [#456](https://github.com/crawlab-team/crawlab/issues/456) +- **设置页面优化**. ### Bug 修复 - **无法找到爬虫文件错误**. [#485](https://github.com/crawlab-team/crawlab/issues/485) diff --git a/CHANGELOG.md b/CHANGELOG.md index 056d6c57..9cd931ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ # 0.4.5 (unkown) ### Features / Enhancement - **Interactive Tutorial**. Guide users through the main functionalities of Crawlab. -- **Global Environment Variables**. Allow users to set global environment variables, which will be passed into all spider programs. +- **Global Environment Variables**. Allow users to set global environment variables, which will be passed into all spider programs. [#177](https://github.com/crawlab-team/crawlab/issues/177) +- **Project**. Allow users to link spiders to projects. [#316](https://github.com/crawlab-team/crawlab/issues/316) +- **User Admin Optimization**. Restrict privilleges of admin users. [#456](https://github.com/crawlab-team/crawlab/issues/456) +- **Setting Page Optimization**. ### Bug Fixes - **Unable to find spider file error**. [#485](https://github.com/crawlab-team/crawlab/issues/485) diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go index d9e085d2..054ee2fe 100644 --- a/backend/entity/config_spider.go +++ b/backend/entity/config_spider.go @@ -1,12 +1,22 @@ package entity type ConfigSpiderData struct { - Version string `yaml:"version" json:"version"` + // 通用 + Name string `yaml:"name" json:"name"` + DisplayName string `yaml:"display_name" json:"display_name"` + Col string `yaml:"col" json:"col"` + Remark string `yaml:"remark" json:"remark"` + Type string `yaml:"type" bson:"type"` + + // 可配置爬虫 Engine string `yaml:"engine" json:"engine"` StartUrl string `yaml:"start_url" json:"start_url"` StartStage string `yaml:"start_stage" json:"start_stage"` Stages []Stage `yaml:"stages" json:"stages"` Settings map[string]string `yaml:"settings" json:"settings"` + + // 自定义爬虫 + Cmd string `yaml:"cmd" json:"cmd"` } type Stage struct { diff --git a/backend/main.go b/backend/main.go index 4a84462d..ab0d0e7b 100644 --- a/backend/main.go +++ b/backend/main.go @@ -39,7 +39,6 @@ func main() { log.SetLevelFromString(logLevel) } log.Info("initialized log config successfully") - if viper.GetString("log.isDeletePeriodically") == "Y" { err := services.InitDeleteLogPeriodically() if err != nil { @@ -74,8 +73,24 @@ func main() { debug.PrintStack() panic(err) } + log.Info("initialized schedule successfully") + + // 初始化用户服务 + if err := services.InitUserService(); err != nil { + log.Error("init user service error:" + err.Error()) + debug.PrintStack() + panic(err) + } + log.Info("initialized user service successfully") + + // 初始化依赖服务 + if err := services.InitDepsFetcher(); err != nil { + log.Error("init dependency fetcher error:" + err.Error()) + debug.PrintStack() + panic(err) + } + log.Info("initialized dependency fetcher successfully") } - log.Info("initialized schedule successfully") // 初始化任务执行器 if err := services.InitTaskExecutor(); err != nil { @@ -100,22 +115,6 @@ func main() { } log.Info("initialized spider service successfully") - // 初始化用户服务 - if err := services.InitUserService(); err != nil { - log.Error("init user service error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized user service successfully") - - // 初始化依赖服务 - if err := services.InitDepsFetcher(); err != nil { - log.Error("init dependency fetcher error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized dependency fetcher successfully") - // 初始化RPC服务 if err := services.InitRpcService(); err != nil { log.Error("init rpc service error:" + err.Error()) @@ -224,10 +223,18 @@ func main() { } // 全局变量 { - authGroup.POST("/variable", routes.PostVariable) // 新增 - authGroup.PUT("/variable/:id", routes.PutVariable) //修改 - authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除 authGroup.GET("/variables", routes.GetVariableList) // 列表 + authGroup.PUT("/variable", routes.PutVariable) // 新增 + authGroup.POST("/variable/:id", routes.PostVariable) //修改 + authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除 + } + // 项目 + { + authGroup.GET("/projects", routes.GetProjectList) // 列表 + authGroup.GET("/projects/tags", routes.GetProjectTags) // 项目标签 + authGroup.PUT("/projects", routes.PutProject) //修改 + authGroup.POST("/projects/:id", routes.PostProject) // 新增 + authGroup.DELETE("/projects/:id", routes.DeleteProject) //删除 } // 统计数据 authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 diff --git a/backend/model/project.go b/backend/model/project.go new file mode 100644 index 00000000..92c72655 --- /dev/null +++ b/backend/model/project.go @@ -0,0 +1,146 @@ +package model + +import ( + "crawlab/constants" + "crawlab/database" + "github.com/apex/log" + "github.com/globalsign/mgo/bson" + "runtime/debug" + "time" +) + +type Project struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Tags []string `json:"tags" bson:"tags"` + + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + + // 前端展示 + Spiders []Spider `json:"spiders" bson:"spiders"` +} + +func (p *Project) Save() error { + s, c := database.GetCol("projects") + defer s.Close() + + p.UpdateTs = time.Now() + + if err := c.UpdateId(p.Id, p); err != nil { + debug.PrintStack() + return err + } + return nil +} + +func (p *Project) Add() error { + s, c := database.GetCol("projects") + defer s.Close() + + p.Id = bson.NewObjectId() + p.UpdateTs = time.Now() + p.CreateTs = time.Now() + if err := c.Insert(p); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +func (p *Project) GetSpiders() ([]Spider, error) { + s, c := database.GetCol("spiders") + defer s.Close() + + var query interface{} + if p.Id.Hex() == constants.ObjectIdNull { + query = bson.M{ + "$or": []bson.M{ + {"project_id": p.Id}, + {"project_id": bson.M{"$exists": false}}, + }, + } + } else { + query = bson.M{"project_id": p.Id} + } + + var spiders []Spider + if err := c.Find(query).All(&spiders); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return spiders, err + } + + return spiders, nil +} + +func GetProject(id bson.ObjectId) (Project, error) { + s, c := database.GetCol("projects") + defer s.Close() + var p Project + if err := c.Find(bson.M{"_id": id}).One(&p); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return p, err + } + return p, nil +} + +func GetProjectList(filter interface{}, skip int, sortKey string) ([]Project, error) { + s, c := database.GetCol("projects") + defer s.Close() + + var projects []Project + if err := c.Find(filter).Skip(skip).Limit(constants.Infinite).Sort(sortKey).All(&projects); err != nil { + debug.PrintStack() + return projects, err + } + return projects, nil +} + +func GetProjectListTotal(filter interface{}) (int, error) { + s, c := database.GetCol("projects") + defer s.Close() + + var result int + result, err := c.Find(filter).Count() + if err != nil { + return result, err + } + return result, nil +} + +func UpdateProject(id bson.ObjectId, item Project) error { + s, c := database.GetCol("projects") + defer s.Close() + + var result Project + if err := c.FindId(id).One(&result); err != nil { + debug.PrintStack() + return err + } + + if err := item.Save(); err != nil { + return err + } + return nil +} + +func RemoveProject(id bson.ObjectId) error { + s, c := database.GetCol("projects") + defer s.Close() + + var result User + if err := c.FindId(id).One(&result); err != nil { + return err + } + + if err := c.RemoveId(id); err != nil { + return err + } + + return nil +} diff --git a/backend/model/spider.go b/backend/model/spider.go index 3026a66b..2baeb6ed 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -32,6 +32,7 @@ type Spider struct { Envs []Env `json:"envs" bson:"envs"` // 环境变量 Remark string `json:"remark" bson:"remark"` // 备注 Src string `json:"src" bson:"src"` // 源码位置 + ProjectId bson.ObjectId `json:"project_id" bson:"project_id"` // 项目ID // 自定义爬虫 Cmd string `json:"cmd" bson:"cmd"` // 执行命令 @@ -56,6 +57,11 @@ func (spider *Spider) Save() error { spider.UpdateTs = time.Now() + // 兼容没有项目ID的爬虫 + if spider.ProjectId.Hex() == "" { + spider.ProjectId = bson.ObjectIdHex(constants.ObjectIdNull) + } + if err := c.UpdateId(spider.Id, spider); err != nil { debug.PrintStack() return err @@ -162,7 +168,7 @@ func GetSpiderByName(name string) Spider { defer s.Close() var result Spider - if err := c.Find(bson.M{"name": name}).One(&result); err != nil { + if err := c.Find(bson.M{"name": name}).One(&result); err != nil && err != mgo.ErrNotFound { log.Errorf("get spider error: %s, spider_name: %s", err.Error(), name) //debug.PrintStack() return result diff --git a/backend/routes/projects.go b/backend/routes/projects.go new file mode 100644 index 00000000..34b2d7f4 --- /dev/null +++ b/backend/routes/projects.go @@ -0,0 +1,190 @@ +package routes + +import ( + "crawlab/constants" + "crawlab/database" + "crawlab/model" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" +) + +func GetProjectList(c *gin.Context) { + tag := c.Query("tag") + + // 筛选条件 + query := bson.M{} + if tag != "" { + query["tags"] = tag + } + + // 获取列表 + projects, err := model.GetProjectList(query, 0, "+_id") + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取总数 + total, err := model.GetProjectListTotal(query) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取每个项目的爬虫列表 + for i, p := range projects { + spiders, err := p.GetSpiders() + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + projects[i].Spiders = spiders + } + + // 获取未被分配的爬虫数量 + if tag == "" { + noProject := model.Project{ + Id: bson.ObjectIdHex(constants.ObjectIdNull), + Name: "No Project", + Description: "Not assigned to any project", + } + spiders, err := noProject.GetSpiders() + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + noProject.Spiders = spiders + projects = append(projects, noProject) + } + + c.JSON(http.StatusOK, ListResponse{ + Status: "ok", + Message: "success", + Data: projects, + Total: total, + }) +} + +func PutProject(c *gin.Context) { + // 绑定请求数据 + var p model.Project + if err := c.ShouldBindJSON(&p); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + if err := p.Add(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func PostProject(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + } + + var item model.Project + if err := c.ShouldBindJSON(&item); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + if err := model.UpdateProject(bson.ObjectIdHex(id), item); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func DeleteProject(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } + + // 从数据库中删除该爬虫 + if err := model.RemoveProject(bson.ObjectIdHex(id)); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取相关的爬虫 + var spiders []model.Spider + s, col := database.GetCol("spiders") + defer s.Close() + if err := col.Find(bson.M{"project_id": bson.ObjectIdHex(id)}).All(&spiders); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 将爬虫的项目ID置空 + for _, spider := range spiders { + spider.ProjectId = bson.ObjectIdHex(constants.ObjectIdNull) + if err := spider.Save(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func GetProjectTags(c *gin.Context) { + type Result struct { + Tag string `json:"tag" bson:"tag"` + } + + s, col := database.GetCol("projects") + defer s.Close() + + pipeline := []bson.M{ + { + "$unwind": "$tags", + }, + { + "$group": bson.M{ + "_id": "$tags", + }, + }, + { + "$sort": bson.M{ + "_id": 1, + }, + }, + { + "$addFields": bson.M{ + "tag": "$_id", + }, + }, + } + + var items []Result + if err := col.Pipe(pipeline).All(&items); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: items, + }) +} diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 4adfb707..2b6dfd63 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -30,6 +30,7 @@ func GetSpiderList(c *gin.Context) { pageNum, _ := c.GetQuery("page_num") pageSize, _ := c.GetQuery("page_size") keyword, _ := c.GetQuery("keyword") + pid, _ := c.GetQuery("project_id") t, _ := c.GetQuery("type") sortKey, _ := c.GetQuery("sort_key") sortDirection, _ := c.GetQuery("sort_direction") @@ -41,6 +42,16 @@ func GetSpiderList(c *gin.Context) { if t != "" && t != "all" { filter["type"] = t } + if pid == "" { + // do nothing + } else if pid == constants.ObjectIdNull { + filter["$or"] = []bson.M{ + {"project_id": bson.ObjectIdHex(pid)}, + {"project_id": bson.M{"$exists": false}}, + } + } else { + filter["project_id"] = bson.ObjectIdHex(pid) + } // 排序 sortStr := "-_id" diff --git a/backend/routes/variable.go b/backend/routes/variable.go index 56f51ed7..c35c16ab 100644 --- a/backend/routes/variable.go +++ b/backend/routes/variable.go @@ -8,7 +8,7 @@ import ( ) // 新增 -func PostVariable(c *gin.Context) { +func PutVariable(c *gin.Context) { var variable model.Variable if err := c.ShouldBindJSON(&variable); err != nil { HandleError(http.StatusBadRequest, c, err) @@ -22,7 +22,7 @@ func PostVariable(c *gin.Context) { } // 修改 -func PutVariable(c *gin.Context) { +func PostVariable(c *gin.Context) { var id = c.Param("id") var variable model.Variable if err := c.ShouldBindJSON(&variable); err != nil { diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go index fe0a3da1..29e1c2ca 100644 --- a/backend/services/config_spider.go +++ b/backend/services/config_spider.go @@ -6,6 +6,7 @@ import ( "crawlab/entity" "crawlab/model" "crawlab/model/config_spider" + "crawlab/services/spider_handler" "crawlab/utils" "errors" "fmt" @@ -227,6 +228,17 @@ func ProcessSpiderFilesFromConfigData(spider model.Spider, configData entity.Con spider.FileId = fid _ = spider.Save() + // 获取爬虫同步实例 + spiderSync := spider_handler.SpiderSync{ + Spider: spider, + } + + // 获取gfFile + gfFile2 := model.GetGridFs(spider.FileId) + + // 生成MD5 + spiderSync.CreateMd5File(gfFile2.Md5) + return nil } diff --git a/backend/services/spider.go b/backend/services/spider.go index a03d4dc8..48777042 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -14,7 +14,10 @@ import ( "github.com/globalsign/mgo/bson" "github.com/satori/go.uuid" "github.com/spf13/viper" + "gopkg.in/yaml.v2" + "io/ioutil" "os" + "path" "path/filepath" "runtime/debug" ) @@ -264,5 +267,108 @@ func InitSpiderService() error { // 启动定时任务 c.Start() + if model.IsMaster() { + // 添加Demo爬虫 + templateSpidersDir := "../spiders" + for _, info := range utils.ListDir(templateSpidersDir) { + if !info.IsDir() { + continue + } + spiderName := info.Name() + + // 如果爬虫在数据库中不存在，则添加 + spider := model.GetSpiderByName(spiderName) + if spider.Name != "" { + // 存在同名爬虫，跳过 + continue + } + + // 拷贝爬虫 + templateSpiderPath := path.Join(templateSpidersDir, spiderName) + spiderPath := path.Join(viper.GetString("spider.path"), spiderName) + if utils.Exists(spiderPath) { + utils.RemoveFiles(spiderPath) + } + if err := utils.CopyDir(templateSpiderPath, spiderPath); err != nil { + log.Errorf("copy error: " + err.Error()) + debug.PrintStack() + continue + } + + // 构造配置数据 + configData := entity.ConfigSpiderData{} + + // 读取YAML文件 + yamlFile, err := ioutil.ReadFile(path.Join(spiderPath, "Spiderfile")) + if err != nil { + log.Errorf("read yaml error: " + err.Error()) + //debug.PrintStack() + continue + } + + // 反序列化 + if err := yaml.Unmarshal(yamlFile, &configData); err != nil { + log.Errorf("unmarshal error: " + err.Error()) + debug.PrintStack() + continue + } + + if configData.Type == constants.Customized { + // 添加该爬虫到数据库 + spider = model.Spider{ + Id: bson.NewObjectId(), + Name: configData.Name, + DisplayName: configData.DisplayName, + Type: constants.Customized, + Col: configData.Col, + Src: spiderPath, + Remark: configData.Remark, + ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), + FileId: bson.ObjectIdHex(constants.ObjectIdNull), + Cmd: configData.Cmd, + } + if err := spider.Add(); err != nil { + log.Errorf("add spider error: " + err.Error()) + debug.PrintStack() + continue + } + + // 上传爬虫到GridFS + if err := UploadSpiderToGridFsFromMaster(spider); err != nil { + log.Errorf("upload spider error: " + err.Error()) + debug.PrintStack() + continue + } + } else if configData.Type == constants.Configurable || configData.Type == "config" { + // 添加该爬虫到数据库 + spider = model.Spider{ + Id: bson.NewObjectId(), + Name: configData.Name, + DisplayName: configData.DisplayName, + Type: constants.Configurable, + Col: configData.Col, + Src: spiderPath, + Remark: configData.Remark, + ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), + FileId: bson.ObjectIdHex(constants.ObjectIdNull), + Config: configData, + } + if err := spider.Add(); err != nil { + log.Errorf("add spider error: " + err.Error()) + debug.PrintStack() + continue + } + + // 根据序列化后的数据处理爬虫文件 + if err := ProcessSpiderFilesFromConfigData(spider, configData); err != nil { + log.Errorf("add spider error: " + err.Error()) + debug.PrintStack() + continue + } + } + } + + } + return nil } diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 93fa9c12..48b70a12 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -18,6 +18,20 @@ + + + + + @@ -127,6 +141,9 @@ export default { ...mapGetters('user', [ 'token' ]), + ...mapState('project', [ + 'projectList' + ]), isShowRun () { if (this.spiderForm.type === 'customized') { return !!this.spiderForm.cmd @@ -180,6 +197,15 @@ export default { onUploadError () { this.uploadLoading = false } + }, + async created () { + // fetch project list + await this.$store.dispatch('project/getProjectList') + + // 兼容项目ID + if (!this.spiderForm.project_id) { + this.$set(this.spiderForm, 'project_id', '000000000000000000000000') + } } } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index c997248e..8a3fb800 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -12,6 +12,7 @@ export default { 'Deploys': '部署', 'Sites': '网站', 'Setting': '设置', + 'Project': '项目', // 标签 'Overview': '概览', @@ -71,6 +72,7 @@ export default { 'Create Directory': '新建目录', 'Create File': '新建文件', 'Add Node': '添加节点', + 'Add Project': '添加项目', // 主页 'Total Tasks': '总任务数', @@ -217,6 +219,14 @@ export default { // 部署 'Time': '时间', + // 项目 + 'All Tags': '全部标签', + 'Project Name': '项目名称', + 'Project Description': '项目描述', + 'Tags': '标签', + 'Enter Tags': '输入标签', + 'No Project': '无项目', + // 定时任务 'Schedule Name': '定时任务名称', 'Schedule Description': '定时任务描述', @@ -245,6 +255,9 @@ export default { 'Home Page Response Time (sec)': '首页响应时间(秒)', 'Home Page Response Status Code': '首页响应状态码', + // 用户 + 'Super Admin': '超级管理员', + // 文件 'Choose Folder': '选择文件', 'File': '文件', @@ -350,7 +363,7 @@ export default { 'Username': '用户名', 'Password': '密码', 'Confirm Password': '确认密码', - 'normal': '正常用户', + 'normal': '普通用户', 'admin': '管理用户', 'Role': '角色', 'Edit User': '更改用户', diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index a4ba50e1..69b7f35b 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -47,6 +47,25 @@ export const constantRouterMap = [ } ] }, + { + path: '/projects', + component: Layout, + meta: { + title: 'Project', + icon: 'fa fa-gear' + }, + children: [ + { + path: '', + name: 'Project', + component: () => import('../views/project/ProjectList'), + meta: { + title: 'Project', + icon: 'fa fa-code-fork' + } + } + ] + }, { path: '/spiders', component: Layout, diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index 34c98b4a..4fcb86db 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -16,6 +16,7 @@ import stats from './modules/stats' import setting from './modules/setting' import version from './modules/version' import tour from './modules/tour' +import project from './modules/project' import getters from './getters' Vue.use(Vuex) @@ -37,6 +38,7 @@ const store = new Vuex.Store({ setting, version, tour, + project, // 统计 stats }, diff --git a/frontend/src/store/modules/project.js b/frontend/src/store/modules/project.js new file mode 100644 index 00000000..0c6b504f --- /dev/null +++ b/frontend/src/store/modules/project.js @@ -0,0 +1,60 @@ +import request from '../../api/request' + +const state = { + projectForm: {}, + projectList: [], + projectTags: [] +} + +const getters = {} + +const mutations = { + SET_PROJECT_FORM: (state, value) => { + state.projectForm = value + }, + SET_PROJECT_LIST: (state, value) => { + state.projectList = value + }, + SET_PROJECT_TAGS: (state, value) => { + state.projectTags = value + } +} + +const actions = { + getProjectList ({ state, commit }, payload) { + return request.get('/projects', payload) + .then(response => { + if (response.data.data) { + commit('SET_PROJECT_LIST', response.data.data.map(d => { + if (!d.spiders) d.spiders = [] + return d + })) + } + }) + }, + getProjectTags ({ state, commit }) { + return request.get('/projects/tags') + .then(response => { + if (response.data.data) { + commit('SET_PROJECT_TAGS', response.data.data.map(d => d.tag)) + } + }) + }, + addProject ({ state }) { + return request.put('/projects', state.projectForm) + }, + editProject ({ state }, id) { + return request.post(`/projects/${id}`, state.projectForm) + }, + removeProject ({ state }, id) { + return request.delete(`/projects/${id}`) + } +} + +export default { + namespaced: true, + state, + getters, + mutations, + actions +} diff --git a/frontend/src/store/modules/user.js b/frontend/src/store/modules/user.js index 3324ba15..4bb6e918 100644 --- a/frontend/src/store/modules/user.js +++ b/frontend/src/store/modules/user.js @@ -156,7 +156,7 @@ const user = { }, // 新增全局变量 addGlobalVariable ({ commit, state }) { - return request.post(`/variable`, state.globalVariableForm) + return request.put(`/variable`, state.globalVariableForm) .then(() => { state.globalVariableForm = {} }) diff --git a/frontend/src/views/layout/components/Sidebar/SidebarItem.vue b/frontend/src/views/layout/components/Sidebar/SidebarItem.vue index 9c525c24..983134ad 100644 --- a/frontend/src/views/layout/components/Sidebar/SidebarItem.vue +++ b/frontend/src/views/layout/components/Sidebar/SidebarItem.vue @@ -101,3 +101,10 @@ export default { } } + + diff --git a/frontend/src/views/project/ProjectList.vue b/frontend/src/views/project/ProjectList.vue new file mode 100644 index 00000000..b282c82d --- /dev/null +++ b/frontend/src/views/project/ProjectList.vue @@ -0,0 +1,330 @@ + + + + + diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index c7107f11..bbfbee75 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -58,6 +58,20 @@ + + + + + @@ -104,6 +118,20 @@ + + + + + @@ -147,7 +175,29 @@ - + + + + + + + @@ -335,6 +385,7 @@ export default { crawlConfirmDialogVisible: false, activeSpiderId: undefined, filter: { + project_id: '', keyword: '', type: 'all' }, @@ -491,6 +542,9 @@ export default { ...mapGetters('user', [ 'token' ]), + ...mapState('project', [ + 'projectList' + ]), uploadForm () { return { name: this.spiderForm.name, @@ -517,7 +571,12 @@ export default { this.getList() }, onAdd () { + let projectId = '000000000000000000000000' + if (this.filter.project_id) { + projectId = this.filter.project_id + } this.$store.commit('spider/SET_SPIDER_FORM', { + project_id: projectId, template: this.templateList[0] }) this.addDialogVisible = true @@ -737,14 +796,20 @@ export default { sort_key: this.sort.sortKey, sort_direction: this.sort.sortDirection, keyword: this.filter.keyword, - type: this.filter.type + type: this.filter.type, + project_id: this.filter.project_id } await this.$store.dispatch('spider/getSpiderList', params) } }, async created () { - // fetch spider types - // await this.getTypes() + // fetch project list + await this.$store.dispatch('project/getProjectList') + + // project id + if (this.$route.params.project_id) { + this.filter.project_id = this.$route.params.project_id + } // fetch spider list await this.getList() diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index f52597ca..39aca864 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -137,6 +137,7 @@ export default { }, computed: { ...mapState('task', [ + 'taskForm', 'taskResultsData', 'taskResultsTotalCount' ]), @@ -164,6 +165,9 @@ export default { set (value) { this.$store.commit('task/SET_RESULTS_PAGE_SIZE', value) } + }, + isRunning () { + return ['pending', 'running'].includes(this.taskForm.status) } }, methods: { @@ -197,6 +201,9 @@ export default { this.getTaskLog() this.handle = setInterval(() => { + if (!this.isRunning) return + this.$store.dispatch('task/getTaskData', this.$route.params.id) + this.$store.dispatch('task/getTaskResults', this.$route.params.id) this.getTaskLog() }, 5000) }, diff --git a/frontend/src/views/user/UserList.vue b/frontend/src/views/user/UserList.vue index 9389f08c..6e03e0b1 100644 --- a/frontend/src/views/user/UserList.vue +++ b/frontend/src/views/user/UserList.vue @@ -1,7 +1,7 @@