加入项目管理

backend/main.go

@@ -224,10 +224,17 @@ func main() {
 			}
 			// 全局变量
 			{
-				authGroup.POST("/variable", routes.PostVariable)         // 新增
-				authGroup.PUT("/variable/:id", routes.PutVariable)       //修改
-				authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除
 				authGroup.GET("/variables", routes.GetVariableList)      // 列表
+				authGroup.PUT("/variable", routes.PutVariable)           // 新增
+				authGroup.POST("/variable/:id", routes.PostVariable)     //修改
+				authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除
+			}
+			// 项目
+			{
+				authGroup.GET("/projects", routes.GetProjectList)       // 列表
+				authGroup.PUT("/projects", routes.PutProject)           //修改
+				authGroup.POST("/projects/:id", routes.PostProject)     // 新增
+				authGroup.DELETE("/projects/:id", routes.DeleteProject) //删除
 			}
 			// 统计数据
 			authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据

backend/model/project.go

@@ -0,0 +1,146 @@
+package model
+
+import (
+	"crawlab/constants"
+	"crawlab/database"
+	"github.com/apex/log"
+	"github.com/globalsign/mgo/bson"
+	"runtime/debug"
+	"time"
+)
+
+type Project struct {
+	Id          bson.ObjectId `json:"_id" bson:"_id"`
+	Name        string        `json:"name" bson:"name"`
+	Description string        `json:"description" bson:"description"`
+	Tags        []string      `json:"tags" bson:"tags"`
+
+	CreateTs time.Time `json:"create_ts" bson:"create_ts"`
+	UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
+
+	// 前端展示
+	Spiders []Spider `json:"spiders" bson:"spiders"`
+}
+
+func (p *Project) Save() error {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	p.UpdateTs = time.Now()
+
+	if err := c.UpdateId(p.Id, p); err != nil {
+		debug.PrintStack()
+		return err
+	}
+	return nil
+}
+
+func (p *Project) Add() error {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	p.Id = bson.NewObjectId()
+	p.UpdateTs = time.Now()
+	p.CreateTs = time.Now()
+	if err := c.Insert(p); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	return nil
+}
+
+func (p *Project) GetSpiders() ([]Spider, error) {
+	s, c := database.GetCol("spiders")
+	defer s.Close()
+
+	var query interface{}
+	if p.Id.Hex() == constants.ObjectIdNull {
+		query = bson.M{
+			"$or": []bson.M{
+				{"project_id": p.Id},
+				{"project_id": bson.M{"$exists": false}},
+			},
+		}
+	} else {
+		query = bson.M{"project_id": p.Id}
+	}
+
+	var spiders []Spider
+	if err := c.Find(query).All(&spiders); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return spiders, err
+	}
+
+	return spiders, nil
+}
+
+func GetProject(id bson.ObjectId) (Project, error) {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+	var p Project
+	if err := c.Find(bson.M{"_id": id}).One(&p); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return p, err
+	}
+	return p, nil
+}
+
+func GetProjectList(filter interface{}, skip int, sortKey string) ([]Project, error) {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	var projects []Project
+	if err := c.Find(filter).Skip(skip).Limit(constants.Infinite).Sort(sortKey).All(&projects); err != nil {
+		debug.PrintStack()
+		return projects, err
+	}
+	return projects, nil
+}
+
+func GetProjectListTotal(filter interface{}) (int, error) {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	var result int
+	result, err := c.Find(filter).Count()
+	if err != nil {
+		return result, err
+	}
+	return result, nil
+}
+
+func UpdateProject(id bson.ObjectId, item Project) error {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	var result Project
+	if err := c.FindId(id).One(&result); err != nil {
+		debug.PrintStack()
+		return err
+	}
+
+	if err := item.Save(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func RemoveProject(id bson.ObjectId) error {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	var result User
+	if err := c.FindId(id).One(&result); err != nil {
+		return err
+	}
+
+	if err := c.RemoveId(id); err != nil {
+		return err
+	}
+
+	return nil
+}

backend/routes/projects.go

@@ -0,0 +1,119 @@
+package routes
+
+import (
+	"crawlab/constants"
+	"crawlab/model"
+	"github.com/gin-gonic/gin"
+	"github.com/globalsign/mgo/bson"
+	"net/http"
+)
+
+func GetProjectList(c *gin.Context) {
+	// 获取列表
+	projects, err := model.GetProjectList(nil, 0, "+_id")
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 获取总数
+	total, err := model.GetProjectListTotal(nil)
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 获取每个项目的爬虫列表
+	for i, p := range projects {
+		spiders, err := p.GetSpiders()
+		if err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+		projects[i].Spiders = spiders
+	}
+
+	// 获取未被分配的爬虫数量
+	noProject := model.Project{
+		Id:          bson.ObjectIdHex(constants.ObjectIdNull),
+		Name:        "No Project",
+		Description: "Not assigned to any project",
+	}
+	spiders, err := noProject.GetSpiders()
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+	noProject.Spiders = spiders
+	projects = append(projects, noProject)
+
+	c.JSON(http.StatusOK, ListResponse{
+		Status:  "ok",
+		Message: "success",
+		Data:    projects,
+		Total:   total,
+	})
+}
+
+func PutProject(c *gin.Context) {
+	// 绑定请求数据
+	var p model.Project
+	if err := c.ShouldBindJSON(&p); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+
+	if err := p.Add(); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func PostProject(c *gin.Context) {
+	id := c.Param("id")
+
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "invalid id")
+	}
+
+	var item model.Project
+	if err := c.ShouldBindJSON(&item); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+
+	if err := model.UpdateProject(bson.ObjectIdHex(id), item); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func DeleteProject(c *gin.Context) {
+	id := c.Param("id")
+
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "invalid id")
+		return
+	}
+
+	// 从数据库中删除该爬虫
+	if err := model.RemoveProject(bson.ObjectIdHex(id)); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}

frontend/src/components/InfoView/SpiderInfoView.vue

@@ -18,6 +18,21 @@
         <el-form-item :label="$t('Spider Name')">
           <el-input v-model="spiderForm.display_name" :placeholder="$t('Spider Name')" :disabled="isView"></el-input>
         </el-form-item>
+        <el-form-item :label="$t('Project')" prop="project_id" required>
+          <el-select
+            v-model="spiderForm.project_id"
+            :placeholder="$t('Project')"
+            filterable
+          >
+            <el-option value="000000000000000000000000" :label="$t('No project')"/>
+            <el-option
+              v-for="p in projectList"
+              :key="p._id"
+              :value="p._id"
+              :label="p.name"
+            />
+          </el-select>
+        </el-form-item>
         <el-form-item :label="$t('Source Folder')">
           <el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
         </el-form-item>
@@ -127,6 +142,9 @@ export default {
     ...mapGetters('user', [
       'token'
     ]),
+    ...mapState('project', [
+      'projectList'
+    ]),
     isShowRun () {
       if (this.spiderForm.type === 'customized') {
         return !!this.spiderForm.cmd
@@ -180,6 +198,15 @@ export default {
     onUploadError () {
       this.uploadLoading = false
     }
+  },
+  async created () {
+    // fetch project list
+    await this.$store.dispatch('project/getProjectList')
+
+    // 兼容项目ID
+    if (!this.spiderForm.project_id) {
+      this.spiderForm.project_id = '000000000000000000000000'
+    }
   }
 }
 </script>

frontend/src/i18n/zh.js

@@ -72,6 +72,7 @@ export default {
   'Create Directory': '新建目录',
   'Create File': '新建文件',
   'Add Node': '添加节点',
+  'Add Project': '添加项目',
 
   // 主页
   'Total Tasks': '总任务数',

frontend/src/store/index.js

@@ -16,6 +16,7 @@ import stats from './modules/stats'
 import setting from './modules/setting'
 import version from './modules/version'
 import tour from './modules/tour'
+import project from './modules/project'
 import getters from './getters'
 
 Vue.use(Vuex)
@@ -37,6 +38,7 @@ const store = new Vuex.Store({
     setting,
     version,
     tour,
+    project,
     // 统计
     stats
   },

frontend/src/store/modules/project.js

@@ -0,0 +1,48 @@
+import request from '../../api/request'
+
+const state = {
+  projectForm: {},
+  projectList: []
+}
+
+const getters = {}
+
+const mutations = {
+  SET_PROJECT_FORM: (state, value) => {
+    state.projectForm = value
+  },
+  SET_PROJECT_LIST: (state, value) => {
+    state.projectList = value
+  }
+}
+
+const actions = {
+  getProjectList ({ state, commit }) {
+    request.get('/projects')
+      .then(response => {
+        if (response.data.data) {
+          commit('SET_PROJECT_LIST', response.data.data.map(d => {
+            if (!d.spiders) d.spiders = []
+            return d
+          }))
+        }
+      })
+  },
+  addProject ({ state }) {
+    request.put('/projects', state.projectForm)
+  },
+  editProject ({ state }, id) {
+    request.post(`/projects/${id}`, state.projectForm)
+  },
+  removeProject ({ state }, id) {
+    request.delete(`/projects/${id}`)
+  }
+}
+
+export default {
+  namespaced: true,
+  state,
+  getters,
+  mutations,
+  actions
+}

frontend/src/store/modules/user.js

@@ -156,7 +156,7 @@ const user = {
     },
     // 新增全局变量
     addGlobalVariable ({ commit, state }) {
-      return request.post(`/variable`, state.globalVariableForm)
+      return request.put(`/variable`, state.globalVariableForm)
         .then(() => {
           state.globalVariableForm = {}
         })

frontend/src/views/project/ProjectList.vue

@@ -1,14 +1,287 @@
 <template>
   <div class="app-container">
+    <!--add popup-->
+    <el-dialog
+      :visible.sync="dialogVisible"
+      width="640px"
+      :before-close="onDialogClose">
+      <el-form label-width="180px"
+               class="add-form"
+               :model="projectForm"
+               :inline-message="true"
+               ref="projectForm"
+               label-position="right">
+        <el-form-item :label="$t('Proejct Name')" prop="name" required>
+          <el-input id="name" v-model="projectForm.name" :placeholder="$t('Project Name')"></el-input>
+        </el-form-item>
+        <el-form-item :label="$t('Project Description')" prop="description">
+          <el-input
+            id="description"
+            type="textarea"
+            v-model="projectForm.description"
+            :placeholder="$t('Project Description')"
+          />
+        </el-form-item>
+        <el-form-item :label="$t('Tags')" prop="tags">
+          <el-select
+            id="tags"
+            v-model="projectForm.tags"
+            :placeholder="$t('Enter Tags')"
+            allow-create
+            filterable
+            multiple
+          >
+          </el-select>
+        </el-form-item>
+      </el-form>
+      <!--取消、保存-->
+      <span slot="footer" class="dialog-footer">
+        <el-button size="small" @click="onDialogClose">{{$t('Cancel')}}</el-button>
+        <el-button id="btn-submit" size="small" type="primary" @click="onAddSubmit">{{$t('Submit')}}</el-button>
+      </span>
+    </el-dialog>
+    <!--./add popup-->
+
+    <div class="action-wrapper">
+      <div class="buttons">
+        <el-button
+          icon="el-icon-plus"
+          type="primary"
+          size="small"
+          @click="onAdd"
+        >
+          {{$t('Add Project')}}
+        </el-button>
+      </div>
+    </div>
+    <div class="content">
+      <div v-if="projectList.length === 0" class="empty-list">
+        {{ $t('You have no projects created. You can create a project by clicking the "Add" button.')}}
+      </div>
+      <ul v-else class="list">
+        <li
+          class="item"
+          v-for="item in projectList"
+          :key="item._id"
+          @click="onView(item)"
+        >
+          <el-card
+            class="item-card"
+          >
+            <i class="btn-edit fa fa-edit" @click="onEdit(item)"></i>
+            <i class="btn-close fa fa-trash-o" @click="onRemove(item)"></i>
+            <el-row>
+              <h4 class="title">{{ item.name }}</h4>
+            </el-row>
+            <el-row>
+              <div class="spider-count">
+                {{$t('Spider Count')}}: {{ item.spiders.length }}
+              </div>
+            </el-row>
+            <el-row class="description-wrapper">
+              <div class="description">
+                {{ item.description }}
+              </div>
+            </el-row>
+            <el-row class="tags-wrapper">
+              <div class="tags">
+                <el-tag
+                  v-for="(tag, index) in item.tags"
+                  :key="index"
+                  size="mini"
+                  class="tag"
+                >
+                  {{ tag }}
+                </el-tag>
+              </div>
+            </el-row>
+          </el-card>
+        </li>
+      </ul>
+    </div>
   </div>
 </template>
 
 <script>
+import {
+  mapState
+} from 'vuex'
+
 export default {
-  name: 'ProjectList'
+  name: 'ProjectList',
+  data () {
+    return {
+      defaultTags: [],
+      dialogVisible: false
+    }
+  },
+  computed: {
+    ...mapState('project', [
+      'projectForm',
+      'projectList'
+    ])
+  },
+  methods: {
+    onDialogClose () {
+      this.dialogVisible = false
+    },
+    onAdd () {
+      this.isEdit = false
+      this.dialogVisible = true
+      this.$store.commit('project/SET_PROJECT_FORM', { tags: [] })
+      this.$st.sendEv('项目', '添加项目')
+    },
+    onAddSubmit () {
+      this.$refs.projectForm.validate(res => {
+        if (res) {
+          const form = JSON.parse(JSON.stringify(this.projectForm))
+          if (this.isEdit) {
+            this.$request.post(`/projects/${this.projectForm._id}`, form).then(response => {
+              if (response.data.error) {
+                this.$message.error(response.data.error)
+                return
+              }
+              this.dialogVisible = false
+              this.$store.dispatch('project/getProjectList')
+              this.$message.success(this.$t('The project has been saved'))
+            })
+          } else {
+            this.$request.put('/projects', form).then(response => {
+              if (response.data.error) {
+                this.$message.error(response.data.error)
+                return
+              }
+              this.dialogVisible = false
+              this.$store.dispatch('project/getProjectList')
+              this.$message.success(this.$t('The project has been added'))
+            })
+          }
+        }
+      })
+      this.$st.sendEv('项目', '提交项目')
+    },
+    onEdit (row) {
+      this.$store.commit('project/SET_PROJECT_FORM', row)
+      this.dialogVisible = true
+      this.isEdit = true
+      this.$st.sendEv('项目', '修改项目')
+    },
+    onRemove (row) {
+      this.$confirm(this.$t('Are you sure to delete the project?'), this.$t('Notification'), {
+        confirmButtonText: this.$t('Confirm'),
+        cancelButtonText: this.$t('Cancel'),
+        type: 'warning'
+      }).then(() => {
+        this.$store.dispatch('project/removeProject', row._id)
+          .then(() => {
+            setTimeout(() => {
+              this.$store.dispatch('project/getProjectList')
+              this.$message.success(this.$t('The project has been removed'))
+            }, 100)
+          })
+      }).catch(() => {
+      })
+      this.$st.sendEv('项目', '删除项目')
+    },
+    onView (row) {
+      this.$router.push({
+        name: 'SpiderList',
+        params: {
+          project_id: row._id
+        }
+      })
+    }
+  },
+  async created () {
+    await this.$store.dispatch('project/getProjectList')
+  }
 }
 </script>
 
 <style scoped>
+  .action-wrapper {
+    text-align: right;
+    padding-bottom: 10px;
+    border-bottom: 1px solid #EBEEF5;
+  }
+
+  .list {
+    margin: 0;
+    padding: 0;
+    list-style: none;
+    display: flex;
+    flex-wrap: wrap;
+  }
+
+  .list .item {
+    width: 320px;
+    margin: 10px;
+  }
+
+  .list .item .item-card {
+    position: relative;
+    cursor: pointer;
+  }
+
+  .list .item .item-card .title {
+    margin: 10px 0 0 0;
+  }
+
+  .list .item .item-card .spider-count {
+    font-size: 12px;
+    color: grey;
+    font-weight: bolder;
+  }
+
+  .list .item .item-card .description-wrapper {
+    padding-bottom: 5px;
+    margin-bottom: 0;
+    border-bottom: 1px solid #EBEEF5;
+  }
+
+  .list .item .item-card .description {
+    font-size: 12px;
+    color: grey;
+  }
+
+  .list .item .item-card .tags {
+    margin-bottom: -5px;
+  }
+
+  .list .item .item-card .tags .tag {
+    margin: 0 5px 5px 0;
+  }
+
+  .list .item .item-card .el-row {
+    margin-bottom: 5px;
+  }
+
+  .list .item .item-card .el-row:last-child {
+    margin-bottom: 0;
+  }
+
+  .list .item .item-card .btn-edit {
+    z-index: 1;
+    color: grey;
+    position: absolute;
+    top: 11px;
+    right: 40px;
+  }
+
+  .list .item .item-card .btn-close {
+    z-index: 1;
+    color: grey;
+    position: absolute;
+    top: 10px;
+    right: 10px;
+  }
+
+  .empty-list {
+    font-size: 24px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: calc(100vh - 240px);
+  }
 
 </style>

frontend/src/views/spider/SpiderList.vue

@@ -58,6 +58,20 @@
             <el-form-item :label="$t('Display Name')" prop="display_name" required>
               <el-input id="display-name" v-model="spiderForm.display_name" :placeholder="$t('Display Name')"/>
             </el-form-item>
+            <el-form-item :label="$t('Project')" prop="project_id" required>
+              <el-select
+                v-model="spiderForm.project_id"
+                :placeholder="$t('Project')"
+                filterable
+              >
+                <el-option
+                  v-for="p in projectList"
+                  :key="p._id"
+                  :value="p._id"
+                  :label="p.name"
+                />
+              </el-select>
+            </el-form-item>
             <el-form-item :label="$t('Execute Command')" prop="cmd" required>
               <el-input id="cmd" v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"/>
             </el-form-item>
@@ -104,6 +118,20 @@
             <el-form-item :label="$t('Display Name')" prop="display_name" required>
               <el-input v-model="spiderForm.display_name" :placeholder="$t('Display Name')"/>
             </el-form-item>
+            <el-form-item :label="$t('Project')" prop="project_id" required>
+              <el-select
+                v-model="spiderForm.project_id"
+                :placeholder="$t('Project')"
+                filterable
+              >
+                <el-option
+                  v-for="p in projectList"
+                  :key="p._id"
+                  :value="p._id"
+                  :label="p.name"
+                />
+              </el-select>
+            </el-form-item>
             <el-form-item :label="$t('Template')" prop="template" required>
               <el-select id="template" v-model="spiderForm.template" :value="spiderForm.template"
                          :placeholder="$t('Template')">
@@ -147,7 +175,29 @@
             <!--              </el-select>-->
             <!--            </el-form-item>-->
             <el-form-item>
-              <el-input clearable @keyup.enter.native="onSearch" size="small" placeholder="名称" v-model="filter.keyword">
+              <el-select
+                v-model="filter.project_id"
+                size="small"
+                :placeholder="$t('Project')"
+                @change="getList"
+              >
+                <el-option value="" :label="$t('All Projects')"/>
+                <el-option
+                  v-for="p in projectList"
+                  :key="p._id"
+                  :value="p._id"
+                  :label="p.name"
+                />
+              </el-select>
+            </el-form-item>
+            <el-form-item>
+              <el-input
+                v-model="filter.keyword"
+                size="small"
+                :placeholder="$t('Spider Name')"
+                clearable
+                @keyup.enter.native="onSearch"
+              >
                 <i slot="suffix" class="el-input__icon el-icon-search"></i>
               </el-input>
             </el-form-item>
@@ -335,6 +385,7 @@ export default {
       crawlConfirmDialogVisible: false,
       activeSpiderId: undefined,
       filter: {
+        project_id: '',
         keyword: '',
         type: 'all'
       },
@@ -491,6 +542,9 @@ export default {
     ...mapGetters('user', [
       'token'
     ]),
+    ...mapState('project', [
+      'projectList'
+    ]),
     uploadForm () {
       return {
         name: this.spiderForm.name,
@@ -518,6 +572,7 @@ export default {
     },
     onAdd () {
       this.$store.commit('spider/SET_SPIDER_FORM', {
+        project_id: '000000000000000000000000',
         template: this.templateList[0]
       })
       this.addDialogVisible = true
@@ -737,20 +792,27 @@ export default {
         sort_key: this.sort.sortKey,
         sort_direction: this.sort.sortDirection,
         keyword: this.filter.keyword,
-        type: this.filter.type
+        type: this.filter.type,
+        project_id: this.filter.project_id
       }
       await this.$store.dispatch('spider/getSpiderList', params)
     }
   },
   async created () {
-    // fetch spider types
-    // await this.getTypes()
+    // fetch project list
+    await this.$store.dispatch('project/getProjectList')
+
+    // project id
+    if (this.$route.params.project_id) {
+      this.filter.project_id = this.$route.params.project_id
+    }
 
     // fetch spider list
     await this.getList()
 
     // fetch template list
     await this.$store.dispatch('spider/getTemplateList')
+
   },
   mounted () {
     const vm = this