diff --git a/backend/main.go b/backend/main.go index 4a84462d..3a4ae251 100644 --- a/backend/main.go +++ b/backend/main.go @@ -224,10 +224,17 @@ func main() { } // 全局变量 { - authGroup.POST("/variable", routes.PostVariable) // 新增 - authGroup.PUT("/variable/:id", routes.PutVariable) //修改 - authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除 authGroup.GET("/variables", routes.GetVariableList) // 列表 + authGroup.PUT("/variable", routes.PutVariable) // 新增 + authGroup.POST("/variable/:id", routes.PostVariable) //修改 + authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除 + } + // 项目 + { + authGroup.GET("/projects", routes.GetProjectList) // 列表 + authGroup.PUT("/projects", routes.PutProject) //修改 + authGroup.POST("/projects/:id", routes.PostProject) // 新增 + authGroup.DELETE("/projects/:id", routes.DeleteProject) //删除 } // 统计数据 authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 diff --git a/backend/model/project.go b/backend/model/project.go new file mode 100644 index 00000000..92c72655 --- /dev/null +++ b/backend/model/project.go @@ -0,0 +1,146 @@ +package model + +import ( + "crawlab/constants" + "crawlab/database" + "github.com/apex/log" + "github.com/globalsign/mgo/bson" + "runtime/debug" + "time" +) + +type Project struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Tags []string `json:"tags" bson:"tags"` + + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + + // 前端展示 + Spiders []Spider `json:"spiders" bson:"spiders"` +} + +func (p *Project) Save() error { + s, c := database.GetCol("projects") + defer s.Close() + + p.UpdateTs = time.Now() + + if err := c.UpdateId(p.Id, p); err != nil { + debug.PrintStack() + return err + } + return nil +} + +func (p *Project) Add() error { + s, c := database.GetCol("projects") + defer s.Close() + + p.Id = bson.NewObjectId() + p.UpdateTs = time.Now() + p.CreateTs = time.Now() + if err := c.Insert(p); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +func (p *Project) GetSpiders() ([]Spider, error) { + s, c := database.GetCol("spiders") + defer s.Close() + + var query interface{} + if p.Id.Hex() == constants.ObjectIdNull { + query = bson.M{ + "$or": []bson.M{ + {"project_id": p.Id}, + {"project_id": bson.M{"$exists": false}}, + }, + } + } else { + query = bson.M{"project_id": p.Id} + } + + var spiders []Spider + if err := c.Find(query).All(&spiders); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return spiders, err + } + + return spiders, nil +} + +func GetProject(id bson.ObjectId) (Project, error) { + s, c := database.GetCol("projects") + defer s.Close() + var p Project + if err := c.Find(bson.M{"_id": id}).One(&p); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return p, err + } + return p, nil +} + +func GetProjectList(filter interface{}, skip int, sortKey string) ([]Project, error) { + s, c := database.GetCol("projects") + defer s.Close() + + var projects []Project + if err := c.Find(filter).Skip(skip).Limit(constants.Infinite).Sort(sortKey).All(&projects); err != nil { + debug.PrintStack() + return projects, err + } + return projects, nil +} + +func GetProjectListTotal(filter interface{}) (int, error) { + s, c := database.GetCol("projects") + defer s.Close() + + var result int + result, err := c.Find(filter).Count() + if err != nil { + return result, err + } + return result, nil +} + +func UpdateProject(id bson.ObjectId, item Project) error { + s, c := database.GetCol("projects") + defer s.Close() + + var result Project + if err := c.FindId(id).One(&result); err != nil { + debug.PrintStack() + return err + } + + if err := item.Save(); err != nil { + return err + } + return nil +} + +func RemoveProject(id bson.ObjectId) error { + s, c := database.GetCol("projects") + defer s.Close() + + var result User + if err := c.FindId(id).One(&result); err != nil { + return err + } + + if err := c.RemoveId(id); err != nil { + return err + } + + return nil +} diff --git a/backend/routes/projects.go b/backend/routes/projects.go new file mode 100644 index 00000000..3607d73a --- /dev/null +++ b/backend/routes/projects.go @@ -0,0 +1,119 @@ +package routes + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" +) + +func GetProjectList(c *gin.Context) { + // 获取列表 + projects, err := model.GetProjectList(nil, 0, "+_id") + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取总数 + total, err := model.GetProjectListTotal(nil) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取每个项目的爬虫列表 + for i, p := range projects { + spiders, err := p.GetSpiders() + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + projects[i].Spiders = spiders + } + + // 获取未被分配的爬虫数量 + noProject := model.Project{ + Id: bson.ObjectIdHex(constants.ObjectIdNull), + Name: "No Project", + Description: "Not assigned to any project", + } + spiders, err := noProject.GetSpiders() + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + noProject.Spiders = spiders + projects = append(projects, noProject) + + c.JSON(http.StatusOK, ListResponse{ + Status: "ok", + Message: "success", + Data: projects, + Total: total, + }) +} + +func PutProject(c *gin.Context) { + // 绑定请求数据 + var p model.Project + if err := c.ShouldBindJSON(&p); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + if err := p.Add(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func PostProject(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + } + + var item model.Project + if err := c.ShouldBindJSON(&item); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + if err := model.UpdateProject(bson.ObjectIdHex(id), item); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func DeleteProject(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } + + // 从数据库中删除该爬虫 + if err := model.RemoveProject(bson.ObjectIdHex(id)); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 93fa9c12..afec2413 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -18,6 +18,21 @@ + + + + + + @@ -127,6 +142,9 @@ export default { ...mapGetters('user', [ 'token' ]), + ...mapState('project', [ + 'projectList' + ]), isShowRun () { if (this.spiderForm.type === 'customized') { return !!this.spiderForm.cmd @@ -180,6 +198,15 @@ export default { onUploadError () { this.uploadLoading = false } + }, + async created () { + // fetch project list + await this.$store.dispatch('project/getProjectList') + + // 兼容项目ID + if (!this.spiderForm.project_id) { + this.spiderForm.project_id = '000000000000000000000000' + } } } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 1a48189f..f7d201dd 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -72,6 +72,7 @@ export default { 'Create Directory': '新建目录', 'Create File': '新建文件', 'Add Node': '添加节点', + 'Add Project': '添加项目', // 主页 'Total Tasks': '总任务数', diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index 34c98b4a..4fcb86db 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -16,6 +16,7 @@ import stats from './modules/stats' import setting from './modules/setting' import version from './modules/version' import tour from './modules/tour' +import project from './modules/project' import getters from './getters' Vue.use(Vuex) @@ -37,6 +38,7 @@ const store = new Vuex.Store({ setting, version, tour, + project, // 统计 stats }, diff --git a/frontend/src/store/modules/project.js b/frontend/src/store/modules/project.js new file mode 100644 index 00000000..bed02795 --- /dev/null +++ b/frontend/src/store/modules/project.js @@ -0,0 +1,48 @@ +import request from '../../api/request' + +const state = { + projectForm: {}, + projectList: [] +} + +const getters = {} + +const mutations = { + SET_PROJECT_FORM: (state, value) => { + state.projectForm = value + }, + SET_PROJECT_LIST: (state, value) => { + state.projectList = value + } +} + +const actions = { + getProjectList ({ state, commit }) { + request.get('/projects') + .then(response => { + if (response.data.data) { + commit('SET_PROJECT_LIST', response.data.data.map(d => { + if (!d.spiders) d.spiders = [] + return d + })) + } + }) + }, + addProject ({ state }) { + request.put('/projects', state.projectForm) + }, + editProject ({ state }, id) { + request.post(`/projects/${id}`, state.projectForm) + }, + removeProject ({ state }, id) { + request.delete(`/projects/${id}`) + } +} + +export default { + namespaced: true, + state, + getters, + mutations, + actions +} diff --git a/frontend/src/store/modules/user.js b/frontend/src/store/modules/user.js index 3324ba15..4bb6e918 100644 --- a/frontend/src/store/modules/user.js +++ b/frontend/src/store/modules/user.js @@ -156,7 +156,7 @@ const user = { }, // 新增全局变量 addGlobalVariable ({ commit, state }) { - return request.post(`/variable`, state.globalVariableForm) + return request.put(`/variable`, state.globalVariableForm) .then(() => { state.globalVariableForm = {} }) diff --git a/frontend/src/views/project/ProjectList.vue b/frontend/src/views/project/ProjectList.vue index deba17a5..726e2202 100644 --- a/frontend/src/views/project/ProjectList.vue +++ b/frontend/src/views/project/ProjectList.vue @@ -1,14 +1,287 @@ diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index c7107f11..f39745bb 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -58,6 +58,20 @@ + + + + + @@ -104,6 +118,20 @@ + + + + + @@ -147,7 +175,29 @@ - + + + + + + + @@ -335,6 +385,7 @@ export default { crawlConfirmDialogVisible: false, activeSpiderId: undefined, filter: { + project_id: '', keyword: '', type: 'all' }, @@ -491,6 +542,9 @@ export default { ...mapGetters('user', [ 'token' ]), + ...mapState('project', [ + 'projectList' + ]), uploadForm () { return { name: this.spiderForm.name, @@ -518,6 +572,7 @@ export default { }, onAdd () { this.$store.commit('spider/SET_SPIDER_FORM', { + project_id: '000000000000000000000000', template: this.templateList[0] }) this.addDialogVisible = true @@ -737,20 +792,27 @@ export default { sort_key: this.sort.sortKey, sort_direction: this.sort.sortDirection, keyword: this.filter.keyword, - type: this.filter.type + type: this.filter.type, + project_id: this.filter.project_id } await this.$store.dispatch('spider/getSpiderList', params) } }, async created () { - // fetch spider types - // await this.getTypes() + // fetch project list + await this.$store.dispatch('project/getProjectList') + + // project id + if (this.$route.params.project_id) { + this.filter.project_id = this.$route.params.project_id + } // fetch spider list await this.getList() // fetch template list await this.$store.dispatch('spider/getTemplateList') + }, mounted () { const vm = this