From 892b7de7f3d85c204c3485b3289cfc84997a2207 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 09:37:19 +0800
Subject: [PATCH 01/14] updated CHANGELOG

---
 CHANGELOG-zh.md | 1 +
 CHANGELOG.md    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md
index bc2b213f..35996c7f 100644
--- a/CHANGELOG-zh.md
+++ b/CHANGELOG-zh.md
@@ -2,6 +2,7 @@
 ### 功能 / 优化
 - **交互式教程**. 引导用户了解 Crawlab 的主要功能.
 - **加入全局环境变量**. 可以设置全局环境变量，然后传入到所有爬虫程序中.
+- **设置页面优化**.
 
 ### Bug 修复
 - **无法找到爬虫文件错误**. [#485](https://github.com/crawlab-team/crawlab/issues/485)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 056d6c57..53164fc3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 ### Features / Enhancement
 - **Interactive Tutorial**. Guide users through the main functionalities of Crawlab.
 - **Global Environment Variables**. Allow users to set global environment variables, which will be passed into all spider programs.
+- **Setting Page Optimization**.
 
 ### Bug Fixes
 - **Unable to find spider file error**. [#485](https://github.com/crawlab-team/crawlab/issues/485)

From d9209ba64d1cf0629e8288bc849437551c82304a Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 09:46:25 +0800
Subject: [PATCH 02/14] fixed
 https://github.com/crawlab-team/crawlab/issues/456

---
 frontend/src/i18n/zh.js              |  5 ++-
 frontend/src/views/user/UserList.vue | 58 ++++++++++++++++++++--------
 2 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index c997248e..ac67dd74 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -245,6 +245,9 @@ export default {
   'Home Page Response Time (sec)': '首页响应时间(秒)',
   'Home Page Response Status Code': '首页响应状态码',
 
+  // 用户
+  'Super Admin': '超级管理员',
+
   // 文件
   'Choose Folder': '选择文件',
   'File': '文件',
@@ -350,7 +353,7 @@ export default {
   'Username': '用户名',
   'Password': '密码',
   'Confirm Password': '确认密码',
-  'normal': '正常用户',
+  'normal': '普通用户',
   'admin': '管理用户',
   'Role': '角色',
   'Edit User': '更改用户',
diff --git a/frontend/src/views/user/UserList.vue b/frontend/src/views/user/UserList.vue
index 9389f08c..6e03e0b1 100644
--- a/frontend/src/views/user/UserList.vue
+++ b/frontend/src/views/user/UserList.vue
@@ -1,7 +1,7 @@
 <template>
   <div class="app-container">
     <!--dialog-->
-    <el-dialog :visible.sync="dialogVisible" :title="$t('Edit User')">
+    <el-dialog :visible.sync="dialogVisible" width="640px" :title="$t('Edit User')">
       <el-form ref="form" :model="userForm" label-width="80px" :rules="rules" inline-message>
         <el-form-item prop="username" :label="$t('Username')" required>
           <el-input v-model="userForm.username" :placeholder="$t('Username')" :disabled="!isAdd"></el-input>
@@ -50,7 +50,10 @@
           :label="$t('Role')"
         >
           <template slot-scope="scope">
-            <el-tag v-if="scope.row.role === 'admin'" type="primary">
+            <el-tag v-if="scope.row.username === 'admin'" type="success">
+              {{ $t('Super Admin') }}
+            </el-tag>
+            <el-tag v-else-if="scope.row.role === 'admin'" type="primary">
               {{ $t(scope.row.role) }}
             </el-tag>
             <el-tag v-else type="warning">
@@ -71,8 +74,20 @@
           fixed="right"
         >
           <template slot-scope="scope">
-            <el-button icon="el-icon-edit" type="warning" size="mini" @click="onEdit(scope.row)"></el-button>
-            <el-button icon="el-icon-delete" type="danger" size="mini" @click="onRemove(scope.row)"></el-button>
+            <el-button
+              v-if="isShowEdit(scope.row)"
+              icon="el-icon-edit"
+              type="warning"
+              size="mini"
+              @click="onEdit(scope.row)"
+            />
+            <el-button
+              v-if="isShowRemove(scope.row)"
+              icon="el-icon-delete"
+              type="danger"
+              size="mini"
+              @click="onRemove(scope.row)"
+            />
           </template>
         </el-table-column>
       </el-table>
@@ -95,7 +110,8 @@
 
 <script>
 import {
-  mapState
+  mapState,
+  mapGetters
 } from 'vuex'
 import dayjs from 'dayjs'
 
@@ -133,6 +149,9 @@ export default {
       'userForm',
       'totalCount'
     ]),
+    ...mapGetters('user', [
+      'userInfo'
+    ]),
     pageSize: {
       get () {
         return this.$store.state.user.pageSize
@@ -219,6 +238,15 @@ export default {
       this.dialogVisible = true
     },
     onValidateEmail (value) {
+    },
+    isShowEdit (row) {
+      if (row.username === 'admin') {
+        return this.userInfo.username === 'admin'
+      }
+      return true
+    },
+    isShowRemove (row) {
+      return row.username !== 'admin'
     }
   },
   created () {
@@ -227,23 +255,21 @@ export default {
 }
 </script>
 
-<style scoped>
+<style lang="scss" scoped>
   .filter {
     display: flex;
     justify-content: space-between;
     margin-bottom: 8px;
 
-  .filter-search {
-    width: 240px;
-  }
+    .filter-search {
+      width: 240px;
+    }
 
-  .right {
-
-  .btn {
-    margin-left: 10px;
-  }
-
-  }
+    .right {
+      .btn {
+        margin-left: 10px;
+      }
+    }
   }
 
   .el-table {

From 7c115486391c30bc4417f17ac3bd64c73157e9c3 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 09:48:07 +0800
Subject: [PATCH 03/14] updated CHANGELOG

---
 CHANGELOG-zh.md | 1 +
 CHANGELOG.md    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md
index 35996c7f..03748273 100644
--- a/CHANGELOG-zh.md
+++ b/CHANGELOG-zh.md
@@ -2,6 +2,7 @@
 ### 功能 / 优化
 - **交互式教程**. 引导用户了解 Crawlab 的主要功能.
 - **加入全局环境变量**. 可以设置全局环境变量，然后传入到所有爬虫程序中.
+- **用户管理优化**. 限制管理用户的权限. [#456](https://github.com/crawlab-team/crawlab/issues/456)
 - **设置页面优化**.
 
 ### Bug 修复
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 53164fc3..2fad8ac0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 ### Features / Enhancement
 - **Interactive Tutorial**. Guide users through the main functionalities of Crawlab.
 - **Global Environment Variables**. Allow users to set global environment variables, which will be passed into all spider programs.
+- **User Admin Optimization**. Restrict privilleges of admin users. [#456](https://github.com/crawlab-team/crawlab/issues/456)
 - **Setting Page Optimization**.
 
 ### Bug Fixes

From 523a5c9c7c2c9db0a46b68c64eb889ea3392f5e4 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 09:50:38 +0800
Subject: [PATCH 04/14] updated CHANGELOG

---
 CHANGELOG-zh.md | 2 +-
 CHANGELOG.md    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md
index 03748273..2f80526d 100644
--- a/CHANGELOG-zh.md
+++ b/CHANGELOG-zh.md
@@ -1,7 +1,7 @@
 # 0.4.5 (unkown)
 ### 功能 / 优化
 - **交互式教程**. 引导用户了解 Crawlab 的主要功能.
-- **加入全局环境变量**. 可以设置全局环境变量，然后传入到所有爬虫程序中.
+- **加入全局环境变量**. 可以设置全局环境变量，然后传入到所有爬虫程序中. [#177](https://github.com/crawlab-team/crawlab/issues/177)
 - **用户管理优化**. 限制管理用户的权限. [#456](https://github.com/crawlab-team/crawlab/issues/456)
 - **设置页面优化**.
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2fad8ac0..b19bc419 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,7 @@
 # 0.4.5 (unkown)
 ### Features / Enhancement
 - **Interactive Tutorial**. Guide users through the main functionalities of Crawlab.
-- **Global Environment Variables**. Allow users to set global environment variables, which will be passed into all spider programs.
+- **Global Environment Variables**. Allow users to set global environment variables, which will be passed into all spider programs. [#177](https://github.com/crawlab-team/crawlab/issues/177)
 - **User Admin Optimization**. Restrict privilleges of admin users. [#456](https://github.com/crawlab-team/crawlab/issues/456)
 - **Setting Page Optimization**.
 

From 99e73c016b4770d066dad4481f0203ebe222521e Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 09:59:47 +0800
Subject: [PATCH 05/14] sidebar optimization

---
 frontend/src/i18n/zh.js                       |  1 +
 frontend/src/router/index.js                  | 19 +++++++++++++++++++
 .../layout/components/Sidebar/SidebarItem.vue |  7 +++++++
 frontend/src/views/project/ProjectList.vue    | 14 ++++++++++++++
 4 files changed, 41 insertions(+)
 create mode 100644 frontend/src/views/project/ProjectList.vue

diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index ac67dd74..1a48189f 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -12,6 +12,7 @@ export default {
   'Deploys': '部署',
   'Sites': '网站',
   'Setting': '设置',
+  'Project': '项目',
 
   // 标签
   'Overview': '概览',
diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js
index a4ba50e1..69b7f35b 100644
--- a/frontend/src/router/index.js
+++ b/frontend/src/router/index.js
@@ -47,6 +47,25 @@ export const constantRouterMap = [
       }
     ]
   },
+  {
+    path: '/projects',
+    component: Layout,
+    meta: {
+      title: 'Project',
+      icon: 'fa fa-gear'
+    },
+    children: [
+      {
+        path: '',
+        name: 'Project',
+        component: () => import('../views/project/ProjectList'),
+        meta: {
+          title: 'Project',
+          icon: 'fa fa-code-fork'
+        }
+      }
+    ]
+  },
   {
     path: '/spiders',
     component: Layout,
diff --git a/frontend/src/views/layout/components/Sidebar/SidebarItem.vue b/frontend/src/views/layout/components/Sidebar/SidebarItem.vue
index 9c525c24..983134ad 100644
--- a/frontend/src/views/layout/components/Sidebar/SidebarItem.vue
+++ b/frontend/src/views/layout/components/Sidebar/SidebarItem.vue
@@ -101,3 +101,10 @@ export default {
   }
 }
 </script>
+
+<style scoped>
+  .menu-wrapper >>> .fa {
+    width: 16px;
+    text-align: center;
+  }
+</style>
diff --git a/frontend/src/views/project/ProjectList.vue b/frontend/src/views/project/ProjectList.vue
new file mode 100644
index 00000000..deba17a5
--- /dev/null
+++ b/frontend/src/views/project/ProjectList.vue
@@ -0,0 +1,14 @@
+<template>
+  <div class="app-container">
+  </div>
+</template>
+
+<script>
+export default {
+  name: 'ProjectList'
+}
+</script>
+
+<style scoped>
+
+</style>

From 9d5d0253a87b8e1483a3cfffc75f7c5e21115a41 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 12:29:28 +0800
Subject: [PATCH 06/14] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E9=A1=B9=E7=9B=AE?=
 =?UTF-8?q?=E7=AE=A1=E7=90=86?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/main.go                               |  13 +-
 backend/model/project.go                      | 146 ++++++++++
 backend/routes/projects.go                    | 119 ++++++++
 .../components/InfoView/SpiderInfoView.vue    |  27 ++
 frontend/src/i18n/zh.js                       |   1 +
 frontend/src/store/index.js                   |   2 +
 frontend/src/store/modules/project.js         |  48 +++
 frontend/src/store/modules/user.js            |   2 +-
 frontend/src/views/project/ProjectList.vue    | 275 +++++++++++++++++-
 frontend/src/views/spider/SpiderList.vue      |  70 ++++-
 10 files changed, 694 insertions(+), 9 deletions(-)
 create mode 100644 backend/model/project.go
 create mode 100644 backend/routes/projects.go
 create mode 100644 frontend/src/store/modules/project.js

diff --git a/backend/main.go b/backend/main.go
index 4a84462d..3a4ae251 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -224,10 +224,17 @@ func main() {
 			}
 			// 全局变量
 			{
-				authGroup.POST("/variable", routes.PostVariable)         // 新增
-				authGroup.PUT("/variable/:id", routes.PutVariable)       //修改
-				authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除
 				authGroup.GET("/variables", routes.GetVariableList)      // 列表
+				authGroup.PUT("/variable", routes.PutVariable)           // 新增
+				authGroup.POST("/variable/:id", routes.PostVariable)     //修改
+				authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除
+			}
+			// 项目
+			{
+				authGroup.GET("/projects", routes.GetProjectList)       // 列表
+				authGroup.PUT("/projects", routes.PutProject)           //修改
+				authGroup.POST("/projects/:id", routes.PostProject)     // 新增
+				authGroup.DELETE("/projects/:id", routes.DeleteProject) //删除
 			}
 			// 统计数据
 			authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
diff --git a/backend/model/project.go b/backend/model/project.go
new file mode 100644
index 00000000..92c72655
--- /dev/null
+++ b/backend/model/project.go
@@ -0,0 +1,146 @@
+package model
+
+import (
+	"crawlab/constants"
+	"crawlab/database"
+	"github.com/apex/log"
+	"github.com/globalsign/mgo/bson"
+	"runtime/debug"
+	"time"
+)
+
+type Project struct {
+	Id          bson.ObjectId `json:"_id" bson:"_id"`
+	Name        string        `json:"name" bson:"name"`
+	Description string        `json:"description" bson:"description"`
+	Tags        []string      `json:"tags" bson:"tags"`
+
+	CreateTs time.Time `json:"create_ts" bson:"create_ts"`
+	UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
+
+	// 前端展示
+	Spiders []Spider `json:"spiders" bson:"spiders"`
+}
+
+func (p *Project) Save() error {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	p.UpdateTs = time.Now()
+
+	if err := c.UpdateId(p.Id, p); err != nil {
+		debug.PrintStack()
+		return err
+	}
+	return nil
+}
+
+func (p *Project) Add() error {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	p.Id = bson.NewObjectId()
+	p.UpdateTs = time.Now()
+	p.CreateTs = time.Now()
+	if err := c.Insert(p); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	return nil
+}
+
+func (p *Project) GetSpiders() ([]Spider, error) {
+	s, c := database.GetCol("spiders")
+	defer s.Close()
+
+	var query interface{}
+	if p.Id.Hex() == constants.ObjectIdNull {
+		query = bson.M{
+			"$or": []bson.M{
+				{"project_id": p.Id},
+				{"project_id": bson.M{"$exists": false}},
+			},
+		}
+	} else {
+		query = bson.M{"project_id": p.Id}
+	}
+
+	var spiders []Spider
+	if err := c.Find(query).All(&spiders); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return spiders, err
+	}
+
+	return spiders, nil
+}
+
+func GetProject(id bson.ObjectId) (Project, error) {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+	var p Project
+	if err := c.Find(bson.M{"_id": id}).One(&p); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return p, err
+	}
+	return p, nil
+}
+
+func GetProjectList(filter interface{}, skip int, sortKey string) ([]Project, error) {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	var projects []Project
+	if err := c.Find(filter).Skip(skip).Limit(constants.Infinite).Sort(sortKey).All(&projects); err != nil {
+		debug.PrintStack()
+		return projects, err
+	}
+	return projects, nil
+}
+
+func GetProjectListTotal(filter interface{}) (int, error) {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	var result int
+	result, err := c.Find(filter).Count()
+	if err != nil {
+		return result, err
+	}
+	return result, nil
+}
+
+func UpdateProject(id bson.ObjectId, item Project) error {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	var result Project
+	if err := c.FindId(id).One(&result); err != nil {
+		debug.PrintStack()
+		return err
+	}
+
+	if err := item.Save(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func RemoveProject(id bson.ObjectId) error {
+	s, c := database.GetCol("projects")
+	defer s.Close()
+
+	var result User
+	if err := c.FindId(id).One(&result); err != nil {
+		return err
+	}
+
+	if err := c.RemoveId(id); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/backend/routes/projects.go b/backend/routes/projects.go
new file mode 100644
index 00000000..3607d73a
--- /dev/null
+++ b/backend/routes/projects.go
@@ -0,0 +1,119 @@
+package routes
+
+import (
+	"crawlab/constants"
+	"crawlab/model"
+	"github.com/gin-gonic/gin"
+	"github.com/globalsign/mgo/bson"
+	"net/http"
+)
+
+func GetProjectList(c *gin.Context) {
+	// 获取列表
+	projects, err := model.GetProjectList(nil, 0, "+_id")
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 获取总数
+	total, err := model.GetProjectListTotal(nil)
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 获取每个项目的爬虫列表
+	for i, p := range projects {
+		spiders, err := p.GetSpiders()
+		if err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+		projects[i].Spiders = spiders
+	}
+
+	// 获取未被分配的爬虫数量
+	noProject := model.Project{
+		Id:          bson.ObjectIdHex(constants.ObjectIdNull),
+		Name:        "No Project",
+		Description: "Not assigned to any project",
+	}
+	spiders, err := noProject.GetSpiders()
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+	noProject.Spiders = spiders
+	projects = append(projects, noProject)
+
+	c.JSON(http.StatusOK, ListResponse{
+		Status:  "ok",
+		Message: "success",
+		Data:    projects,
+		Total:   total,
+	})
+}
+
+func PutProject(c *gin.Context) {
+	// 绑定请求数据
+	var p model.Project
+	if err := c.ShouldBindJSON(&p); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+
+	if err := p.Add(); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func PostProject(c *gin.Context) {
+	id := c.Param("id")
+
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "invalid id")
+	}
+
+	var item model.Project
+	if err := c.ShouldBindJSON(&item); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+
+	if err := model.UpdateProject(bson.ObjectIdHex(id), item); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func DeleteProject(c *gin.Context) {
+	id := c.Param("id")
+
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "invalid id")
+		return
+	}
+
+	// 从数据库中删除该爬虫
+	if err := model.RemoveProject(bson.ObjectIdHex(id)); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue
index 93fa9c12..afec2413 100644
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -18,6 +18,21 @@
         <el-form-item :label="$t('Spider Name')">
           <el-input v-model="spiderForm.display_name" :placeholder="$t('Spider Name')" :disabled="isView"></el-input>
         </el-form-item>
+        <el-form-item :label="$t('Project')" prop="project_id" required>
+          <el-select
+            v-model="spiderForm.project_id"
+            :placeholder="$t('Project')"
+            filterable
+          >
+            <el-option value="000000000000000000000000" :label="$t('No project')"/>
+            <el-option
+              v-for="p in projectList"
+              :key="p._id"
+              :value="p._id"
+              :label="p.name"
+            />
+          </el-select>
+        </el-form-item>
         <el-form-item :label="$t('Source Folder')">
           <el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
         </el-form-item>
@@ -127,6 +142,9 @@ export default {
     ...mapGetters('user', [
       'token'
     ]),
+    ...mapState('project', [
+      'projectList'
+    ]),
     isShowRun () {
       if (this.spiderForm.type === 'customized') {
         return !!this.spiderForm.cmd
@@ -180,6 +198,15 @@ export default {
     onUploadError () {
       this.uploadLoading = false
     }
+  },
+  async created () {
+    // fetch project list
+    await this.$store.dispatch('project/getProjectList')
+
+    // 兼容项目ID
+    if (!this.spiderForm.project_id) {
+      this.spiderForm.project_id = '000000000000000000000000'
+    }
   }
 }
 </script>
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 1a48189f..f7d201dd 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -72,6 +72,7 @@ export default {
   'Create Directory': '新建目录',
   'Create File': '新建文件',
   'Add Node': '添加节点',
+  'Add Project': '添加项目',
 
   // 主页
   'Total Tasks': '总任务数',
diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js
index 34c98b4a..4fcb86db 100644
--- a/frontend/src/store/index.js
+++ b/frontend/src/store/index.js
@@ -16,6 +16,7 @@ import stats from './modules/stats'
 import setting from './modules/setting'
 import version from './modules/version'
 import tour from './modules/tour'
+import project from './modules/project'
 import getters from './getters'
 
 Vue.use(Vuex)
@@ -37,6 +38,7 @@ const store = new Vuex.Store({
     setting,
     version,
     tour,
+    project,
     // 统计
     stats
   },
diff --git a/frontend/src/store/modules/project.js b/frontend/src/store/modules/project.js
new file mode 100644
index 00000000..bed02795
--- /dev/null
+++ b/frontend/src/store/modules/project.js
@@ -0,0 +1,48 @@
+import request from '../../api/request'
+
+const state = {
+  projectForm: {},
+  projectList: []
+}
+
+const getters = {}
+
+const mutations = {
+  SET_PROJECT_FORM: (state, value) => {
+    state.projectForm = value
+  },
+  SET_PROJECT_LIST: (state, value) => {
+    state.projectList = value
+  }
+}
+
+const actions = {
+  getProjectList ({ state, commit }) {
+    request.get('/projects')
+      .then(response => {
+        if (response.data.data) {
+          commit('SET_PROJECT_LIST', response.data.data.map(d => {
+            if (!d.spiders) d.spiders = []
+            return d
+          }))
+        }
+      })
+  },
+  addProject ({ state }) {
+    request.put('/projects', state.projectForm)
+  },
+  editProject ({ state }, id) {
+    request.post(`/projects/${id}`, state.projectForm)
+  },
+  removeProject ({ state }, id) {
+    request.delete(`/projects/${id}`)
+  }
+}
+
+export default {
+  namespaced: true,
+  state,
+  getters,
+  mutations,
+  actions
+}
diff --git a/frontend/src/store/modules/user.js b/frontend/src/store/modules/user.js
index 3324ba15..4bb6e918 100644
--- a/frontend/src/store/modules/user.js
+++ b/frontend/src/store/modules/user.js
@@ -156,7 +156,7 @@ const user = {
     },
     // 新增全局变量
     addGlobalVariable ({ commit, state }) {
-      return request.post(`/variable`, state.globalVariableForm)
+      return request.put(`/variable`, state.globalVariableForm)
         .then(() => {
           state.globalVariableForm = {}
         })
diff --git a/frontend/src/views/project/ProjectList.vue b/frontend/src/views/project/ProjectList.vue
index deba17a5..726e2202 100644
--- a/frontend/src/views/project/ProjectList.vue
+++ b/frontend/src/views/project/ProjectList.vue
@@ -1,14 +1,287 @@
 <template>
   <div class="app-container">
+    <!--add popup-->
+    <el-dialog
+      :visible.sync="dialogVisible"
+      width="640px"
+      :before-close="onDialogClose">
+      <el-form label-width="180px"
+               class="add-form"
+               :model="projectForm"
+               :inline-message="true"
+               ref="projectForm"
+               label-position="right">
+        <el-form-item :label="$t('Proejct Name')" prop="name" required>
+          <el-input id="name" v-model="projectForm.name" :placeholder="$t('Project Name')"></el-input>
+        </el-form-item>
+        <el-form-item :label="$t('Project Description')" prop="description">
+          <el-input
+            id="description"
+            type="textarea"
+            v-model="projectForm.description"
+            :placeholder="$t('Project Description')"
+          />
+        </el-form-item>
+        <el-form-item :label="$t('Tags')" prop="tags">
+          <el-select
+            id="tags"
+            v-model="projectForm.tags"
+            :placeholder="$t('Enter Tags')"
+            allow-create
+            filterable
+            multiple
+          >
+          </el-select>
+        </el-form-item>
+      </el-form>
+      <!--取消、保存-->
+      <span slot="footer" class="dialog-footer">
+        <el-button size="small" @click="onDialogClose">{{$t('Cancel')}}</el-button>
+        <el-button id="btn-submit" size="small" type="primary" @click="onAddSubmit">{{$t('Submit')}}</el-button>
+      </span>
+    </el-dialog>
+    <!--./add popup-->
+
+    <div class="action-wrapper">
+      <div class="buttons">
+        <el-button
+          icon="el-icon-plus"
+          type="primary"
+          size="small"
+          @click="onAdd"
+        >
+          {{$t('Add Project')}}
+        </el-button>
+      </div>
+    </div>
+    <div class="content">
+      <div v-if="projectList.length === 0" class="empty-list">
+        {{ $t('You have no projects created. You can create a project by clicking the "Add" button.')}}
+      </div>
+      <ul v-else class="list">
+        <li
+          class="item"
+          v-for="item in projectList"
+          :key="item._id"
+          @click="onView(item)"
+        >
+          <el-card
+            class="item-card"
+          >
+            <i class="btn-edit fa fa-edit" @click="onEdit(item)"></i>
+            <i class="btn-close fa fa-trash-o" @click="onRemove(item)"></i>
+            <el-row>
+              <h4 class="title">{{ item.name }}</h4>
+            </el-row>
+            <el-row>
+              <div class="spider-count">
+                {{$t('Spider Count')}}: {{ item.spiders.length }}
+              </div>
+            </el-row>
+            <el-row class="description-wrapper">
+              <div class="description">
+                {{ item.description }}
+              </div>
+            </el-row>
+            <el-row class="tags-wrapper">
+              <div class="tags">
+                <el-tag
+                  v-for="(tag, index) in item.tags"
+                  :key="index"
+                  size="mini"
+                  class="tag"
+                >
+                  {{ tag }}
+                </el-tag>
+              </div>
+            </el-row>
+          </el-card>
+        </li>
+      </ul>
+    </div>
   </div>
 </template>
 
 <script>
+import {
+  mapState
+} from 'vuex'
+
 export default {
-  name: 'ProjectList'
+  name: 'ProjectList',
+  data () {
+    return {
+      defaultTags: [],
+      dialogVisible: false
+    }
+  },
+  computed: {
+    ...mapState('project', [
+      'projectForm',
+      'projectList'
+    ])
+  },
+  methods: {
+    onDialogClose () {
+      this.dialogVisible = false
+    },
+    onAdd () {
+      this.isEdit = false
+      this.dialogVisible = true
+      this.$store.commit('project/SET_PROJECT_FORM', { tags: [] })
+      this.$st.sendEv('项目', '添加项目')
+    },
+    onAddSubmit () {
+      this.$refs.projectForm.validate(res => {
+        if (res) {
+          const form = JSON.parse(JSON.stringify(this.projectForm))
+          if (this.isEdit) {
+            this.$request.post(`/projects/${this.projectForm._id}`, form).then(response => {
+              if (response.data.error) {
+                this.$message.error(response.data.error)
+                return
+              }
+              this.dialogVisible = false
+              this.$store.dispatch('project/getProjectList')
+              this.$message.success(this.$t('The project has been saved'))
+            })
+          } else {
+            this.$request.put('/projects', form).then(response => {
+              if (response.data.error) {
+                this.$message.error(response.data.error)
+                return
+              }
+              this.dialogVisible = false
+              this.$store.dispatch('project/getProjectList')
+              this.$message.success(this.$t('The project has been added'))
+            })
+          }
+        }
+      })
+      this.$st.sendEv('项目', '提交项目')
+    },
+    onEdit (row) {
+      this.$store.commit('project/SET_PROJECT_FORM', row)
+      this.dialogVisible = true
+      this.isEdit = true
+      this.$st.sendEv('项目', '修改项目')
+    },
+    onRemove (row) {
+      this.$confirm(this.$t('Are you sure to delete the project?'), this.$t('Notification'), {
+        confirmButtonText: this.$t('Confirm'),
+        cancelButtonText: this.$t('Cancel'),
+        type: 'warning'
+      }).then(() => {
+        this.$store.dispatch('project/removeProject', row._id)
+          .then(() => {
+            setTimeout(() => {
+              this.$store.dispatch('project/getProjectList')
+              this.$message.success(this.$t('The project has been removed'))
+            }, 100)
+          })
+      }).catch(() => {
+      })
+      this.$st.sendEv('项目', '删除项目')
+    },
+    onView (row) {
+      this.$router.push({
+        name: 'SpiderList',
+        params: {
+          project_id: row._id
+        }
+      })
+    }
+  },
+  async created () {
+    await this.$store.dispatch('project/getProjectList')
+  }
 }
 </script>
 
 <style scoped>
+  .action-wrapper {
+    text-align: right;
+    padding-bottom: 10px;
+    border-bottom: 1px solid #EBEEF5;
+  }
+
+  .list {
+    margin: 0;
+    padding: 0;
+    list-style: none;
+    display: flex;
+    flex-wrap: wrap;
+  }
+
+  .list .item {
+    width: 320px;
+    margin: 10px;
+  }
+
+  .list .item .item-card {
+    position: relative;
+    cursor: pointer;
+  }
+
+  .list .item .item-card .title {
+    margin: 10px 0 0 0;
+  }
+
+  .list .item .item-card .spider-count {
+    font-size: 12px;
+    color: grey;
+    font-weight: bolder;
+  }
+
+  .list .item .item-card .description-wrapper {
+    padding-bottom: 5px;
+    margin-bottom: 0;
+    border-bottom: 1px solid #EBEEF5;
+  }
+
+  .list .item .item-card .description {
+    font-size: 12px;
+    color: grey;
+  }
+
+  .list .item .item-card .tags {
+    margin-bottom: -5px;
+  }
+
+  .list .item .item-card .tags .tag {
+    margin: 0 5px 5px 0;
+  }
+
+  .list .item .item-card .el-row {
+    margin-bottom: 5px;
+  }
+
+  .list .item .item-card .el-row:last-child {
+    margin-bottom: 0;
+  }
+
+  .list .item .item-card .btn-edit {
+    z-index: 1;
+    color: grey;
+    position: absolute;
+    top: 11px;
+    right: 40px;
+  }
+
+  .list .item .item-card .btn-close {
+    z-index: 1;
+    color: grey;
+    position: absolute;
+    top: 10px;
+    right: 10px;
+  }
+
+  .empty-list {
+    font-size: 24px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: calc(100vh - 240px);
+  }
 
 </style>
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index c7107f11..f39745bb 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -58,6 +58,20 @@
             <el-form-item :label="$t('Display Name')" prop="display_name" required>
               <el-input id="display-name" v-model="spiderForm.display_name" :placeholder="$t('Display Name')"/>
             </el-form-item>
+            <el-form-item :label="$t('Project')" prop="project_id" required>
+              <el-select
+                v-model="spiderForm.project_id"
+                :placeholder="$t('Project')"
+                filterable
+              >
+                <el-option
+                  v-for="p in projectList"
+                  :key="p._id"
+                  :value="p._id"
+                  :label="p.name"
+                />
+              </el-select>
+            </el-form-item>
             <el-form-item :label="$t('Execute Command')" prop="cmd" required>
               <el-input id="cmd" v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"/>
             </el-form-item>
@@ -104,6 +118,20 @@
             <el-form-item :label="$t('Display Name')" prop="display_name" required>
               <el-input v-model="spiderForm.display_name" :placeholder="$t('Display Name')"/>
             </el-form-item>
+            <el-form-item :label="$t('Project')" prop="project_id" required>
+              <el-select
+                v-model="spiderForm.project_id"
+                :placeholder="$t('Project')"
+                filterable
+              >
+                <el-option
+                  v-for="p in projectList"
+                  :key="p._id"
+                  :value="p._id"
+                  :label="p.name"
+                />
+              </el-select>
+            </el-form-item>
             <el-form-item :label="$t('Template')" prop="template" required>
               <el-select id="template" v-model="spiderForm.template" :value="spiderForm.template"
                          :placeholder="$t('Template')">
@@ -147,7 +175,29 @@
             <!--              </el-select>-->
             <!--            </el-form-item>-->
             <el-form-item>
-              <el-input clearable @keyup.enter.native="onSearch" size="small" placeholder="名称" v-model="filter.keyword">
+              <el-select
+                v-model="filter.project_id"
+                size="small"
+                :placeholder="$t('Project')"
+                @change="getList"
+              >
+                <el-option value="" :label="$t('All Projects')"/>
+                <el-option
+                  v-for="p in projectList"
+                  :key="p._id"
+                  :value="p._id"
+                  :label="p.name"
+                />
+              </el-select>
+            </el-form-item>
+            <el-form-item>
+              <el-input
+                v-model="filter.keyword"
+                size="small"
+                :placeholder="$t('Spider Name')"
+                clearable
+                @keyup.enter.native="onSearch"
+              >
                 <i slot="suffix" class="el-input__icon el-icon-search"></i>
               </el-input>
             </el-form-item>
@@ -335,6 +385,7 @@ export default {
       crawlConfirmDialogVisible: false,
       activeSpiderId: undefined,
       filter: {
+        project_id: '',
         keyword: '',
         type: 'all'
       },
@@ -491,6 +542,9 @@ export default {
     ...mapGetters('user', [
       'token'
     ]),
+    ...mapState('project', [
+      'projectList'
+    ]),
     uploadForm () {
       return {
         name: this.spiderForm.name,
@@ -518,6 +572,7 @@ export default {
     },
     onAdd () {
       this.$store.commit('spider/SET_SPIDER_FORM', {
+        project_id: '000000000000000000000000',
         template: this.templateList[0]
       })
       this.addDialogVisible = true
@@ -737,20 +792,27 @@ export default {
         sort_key: this.sort.sortKey,
         sort_direction: this.sort.sortDirection,
         keyword: this.filter.keyword,
-        type: this.filter.type
+        type: this.filter.type,
+        project_id: this.filter.project_id
       }
       await this.$store.dispatch('spider/getSpiderList', params)
     }
   },
   async created () {
-    // fetch spider types
-    // await this.getTypes()
+    // fetch project list
+    await this.$store.dispatch('project/getProjectList')
+
+    // project id
+    if (this.$route.params.project_id) {
+      this.filter.project_id = this.$route.params.project_id
+    }
 
     // fetch spider list
     await this.getList()
 
     // fetch template list
     await this.$store.dispatch('spider/getTemplateList')
+
   },
   mounted () {
     const vm = this

From 024f9ac3b201712e69c9beeb75de090dd466ed8f Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 13:29:56 +0800
Subject: [PATCH 07/14] finish up project

---
 backend/model/spider.go                  |  6 ++++++
 backend/routes/spider.go                 | 11 +++++++++++
 backend/routes/variable.go               |  4 ++--
 frontend/src/views/spider/SpiderList.vue |  7 +++++--
 4 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/backend/model/spider.go b/backend/model/spider.go
index 3026a66b..475b12e2 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -32,6 +32,7 @@ type Spider struct {
 	Envs        []Env         `json:"envs" bson:"envs"`                 // 环境变量
 	Remark      string        `json:"remark" bson:"remark"`             // 备注
 	Src         string        `json:"src" bson:"src"`                   // 源码位置
+	ProjectId   bson.ObjectId `json:"project_id" bson:"project_id"`     // 项目ID
 
 	// 自定义爬虫
 	Cmd string `json:"cmd" bson:"cmd"` // 执行命令
@@ -56,6 +57,11 @@ func (spider *Spider) Save() error {
 
 	spider.UpdateTs = time.Now()
 
+	// 兼容没有项目ID的爬虫
+	if spider.ProjectId.Hex() == "" {
+		spider.ProjectId = bson.ObjectIdHex(constants.ObjectIdNull)
+	}
+
 	if err := c.UpdateId(spider.Id, spider); err != nil {
 		debug.PrintStack()
 		return err
diff --git a/backend/routes/spider.go b/backend/routes/spider.go
index 4adfb707..2b6dfd63 100644
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -30,6 +30,7 @@ func GetSpiderList(c *gin.Context) {
 	pageNum, _ := c.GetQuery("page_num")
 	pageSize, _ := c.GetQuery("page_size")
 	keyword, _ := c.GetQuery("keyword")
+	pid, _ := c.GetQuery("project_id")
 	t, _ := c.GetQuery("type")
 	sortKey, _ := c.GetQuery("sort_key")
 	sortDirection, _ := c.GetQuery("sort_direction")
@@ -41,6 +42,16 @@ func GetSpiderList(c *gin.Context) {
 	if t != "" && t != "all" {
 		filter["type"] = t
 	}
+	if pid == "" {
+		// do nothing
+	} else if pid == constants.ObjectIdNull {
+		filter["$or"] = []bson.M{
+			{"project_id": bson.ObjectIdHex(pid)},
+			{"project_id": bson.M{"$exists": false}},
+		}
+	} else {
+		filter["project_id"] = bson.ObjectIdHex(pid)
+	}
 
 	// 排序
 	sortStr := "-_id"
diff --git a/backend/routes/variable.go b/backend/routes/variable.go
index 56f51ed7..c35c16ab 100644
--- a/backend/routes/variable.go
+++ b/backend/routes/variable.go
@@ -8,7 +8,7 @@ import (
 )
 
 // 新增
-func PostVariable(c *gin.Context) {
+func PutVariable(c *gin.Context) {
 	var variable model.Variable
 	if err := c.ShouldBindJSON(&variable); err != nil {
 		HandleError(http.StatusBadRequest, c, err)
@@ -22,7 +22,7 @@ func PostVariable(c *gin.Context) {
 }
 
 // 修改
-func PutVariable(c *gin.Context) {
+func PostVariable(c *gin.Context) {
 	var id = c.Param("id")
 	var variable model.Variable
 	if err := c.ShouldBindJSON(&variable); err != nil {
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index f39745bb..bbfbee75 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -571,8 +571,12 @@ export default {
       this.getList()
     },
     onAdd () {
+      let projectId = '000000000000000000000000'
+      if (this.filter.project_id) {
+        projectId = this.filter.project_id
+      }
       this.$store.commit('spider/SET_SPIDER_FORM', {
-        project_id: '000000000000000000000000',
+        project_id: projectId,
         template: this.templateList[0]
       })
       this.addDialogVisible = true
@@ -812,7 +816,6 @@ export default {
 
     // fetch template list
     await this.$store.dispatch('spider/getTemplateList')
-
   },
   mounted () {
     const vm = this

From 18c8dd6253e722968d486fc5c7a47b60e95f74a3 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 14:04:38 +0800
Subject: [PATCH 08/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=A1=B9=E7=9B=AE?=
 =?UTF-8?q?=E7=AE=A1=E7=90=86?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGELOG-zh.md                            |  1 +
 CHANGELOG.md                               |  1 +
 backend/main.go                            |  1 +
 backend/routes/projects.go                 | 79 ++++++++++++++++++----
 frontend/src/i18n/zh.js                    |  3 +
 frontend/src/views/project/ProjectList.vue | 36 ++++++++--
 6 files changed, 103 insertions(+), 18 deletions(-)

diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md
index 2f80526d..bbb63eae 100644
--- a/CHANGELOG-zh.md
+++ b/CHANGELOG-zh.md
@@ -2,6 +2,7 @@
 ### 功能 / 优化
 - **交互式教程**. 引导用户了解 Crawlab 的主要功能.
 - **加入全局环境变量**. 可以设置全局环境变量，然后传入到所有爬虫程序中. [#177](https://github.com/crawlab-team/crawlab/issues/177)
+- **项目**. 允许用户将爬虫关联到项目上. [#316](https://github.com/crawlab-team/crawlab/issues/316)
 - **用户管理优化**. 限制管理用户的权限. [#456](https://github.com/crawlab-team/crawlab/issues/456)
 - **设置页面优化**.
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b19bc419..9cd931ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 ### Features / Enhancement
 - **Interactive Tutorial**. Guide users through the main functionalities of Crawlab.
 - **Global Environment Variables**. Allow users to set global environment variables, which will be passed into all spider programs. [#177](https://github.com/crawlab-team/crawlab/issues/177)
+- **Project**. Allow users to link spiders to projects. [#316](https://github.com/crawlab-team/crawlab/issues/316)
 - **User Admin Optimization**. Restrict privilleges of admin users. [#456](https://github.com/crawlab-team/crawlab/issues/456)
 - **Setting Page Optimization**.
 
diff --git a/backend/main.go b/backend/main.go
index 3a4ae251..d9d44a12 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -232,6 +232,7 @@ func main() {
 			// 项目
 			{
 				authGroup.GET("/projects", routes.GetProjectList)       // 列表
+				authGroup.GET("/projects/tags", routes.GetProjectTags)  // 项目标签
 				authGroup.PUT("/projects", routes.PutProject)           //修改
 				authGroup.POST("/projects/:id", routes.PostProject)     // 新增
 				authGroup.DELETE("/projects/:id", routes.DeleteProject) //删除
diff --git a/backend/routes/projects.go b/backend/routes/projects.go
index 3607d73a..b1f99ab3 100644
--- a/backend/routes/projects.go
+++ b/backend/routes/projects.go
@@ -2,6 +2,7 @@ package routes
 
 import (
 	"crawlab/constants"
+	"crawlab/database"
 	"crawlab/model"
 	"github.com/gin-gonic/gin"
 	"github.com/globalsign/mgo/bson"
@@ -9,15 +10,23 @@ import (
 )
 
 func GetProjectList(c *gin.Context) {
+	tag := c.Query("tag")
+
+	// 筛选条件
+	query := bson.M{}
+	if tag != "" {
+		query["tags"] = tag
+	}
+
 	// 获取列表
-	projects, err := model.GetProjectList(nil, 0, "+_id")
+	projects, err := model.GetProjectList(query, 0, "+_id")
 	if err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return
 	}
 
 	// 获取总数
-	total, err := model.GetProjectListTotal(nil)
+	total, err := model.GetProjectListTotal(query)
 	if err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return
@@ -34,18 +43,20 @@ func GetProjectList(c *gin.Context) {
 	}
 
 	// 获取未被分配的爬虫数量
-	noProject := model.Project{
-		Id:          bson.ObjectIdHex(constants.ObjectIdNull),
-		Name:        "No Project",
-		Description: "Not assigned to any project",
+	if tag == "" {
+		noProject := model.Project{
+			Id:          bson.ObjectIdHex(constants.ObjectIdNull),
+			Name:        "No Project",
+			Description: "Not assigned to any project",
+		}
+		spiders, err := noProject.GetSpiders()
+		if err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+		noProject.Spiders = spiders
+		projects = append(projects, noProject)
 	}
-	spiders, err := noProject.GetSpiders()
-	if err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
-		return
-	}
-	noProject.Spiders = spiders
-	projects = append(projects, noProject)
 
 	c.JSON(http.StatusOK, ListResponse{
 		Status:  "ok",
@@ -117,3 +128,45 @@ func DeleteProject(c *gin.Context) {
 		Message: "success",
 	})
 }
+
+func GetProjectTags(c *gin.Context) {
+	type Result struct {
+		Tag string `json:"tag" bson:"tag"`
+	}
+
+	s, col := database.GetCol("projects")
+	defer s.Close()
+
+	pipeline := []bson.M{
+		{
+			"$unwind": "$tags",
+		},
+		{
+			"$group": bson.M{
+				"_id": "$tags",
+			},
+		},
+		{
+			"$sort": bson.M{
+				"_id": 1,
+			},
+		},
+		{
+			"$addFields": bson.M{
+				"tag": "$_id",
+			},
+		},
+	}
+
+	var items []Result
+	if err := col.Pipe(pipeline).All(&items); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    items,
+	})
+}
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index f7d201dd..6b3bcdfa 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -219,6 +219,9 @@ export default {
   // 部署
   'Time': '时间',
 
+  // 项目
+  'All Tags': '全部标签',
+
   // 定时任务
   'Schedule Name': '定时任务名称',
   'Schedule Description': '定时任务描述',
diff --git a/frontend/src/views/project/ProjectList.vue b/frontend/src/views/project/ProjectList.vue
index 726e2202..e6a11af0 100644
--- a/frontend/src/views/project/ProjectList.vue
+++ b/frontend/src/views/project/ProjectList.vue
@@ -43,7 +43,23 @@
     <!--./add popup-->
 
     <div class="action-wrapper">
-      <div class="buttons">
+      <div class="left">
+        <el-select
+          v-model="filter.tag"
+          size="small"
+          :placeholder="$t('Select Tag')"
+          @change="onFilterChange"
+        >
+          <el-option value="" :label="$t('All Tags')"/>
+          <el-option
+            v-for="tag in projectTags"
+            :key="tag"
+            :label="tag"
+            :value="tag"
+          />
+        </el-select>
+      </div>
+      <div class="right">
         <el-button
           icon="el-icon-plus"
           type="primary"
@@ -112,19 +128,27 @@ export default {
   data () {
     return {
       defaultTags: [],
-      dialogVisible: false
+      dialogVisible: false,
+      filter: {
+        tag: ''
+      }
     }
   },
   computed: {
     ...mapState('project', [
       'projectForm',
-      'projectList'
+      'projectList',
+      'projectTags'
     ])
   },
   methods: {
     onDialogClose () {
       this.dialogVisible = false
     },
+    onFilterChange () {
+      this.$store.dispatch('project/getProjectList', this.filter)
+      this.$st.sendEv('项目', '筛选项目')
+    },
     onAdd () {
       this.isEdit = false
       this.dialogVisible = true
@@ -193,14 +217,16 @@ export default {
     }
   },
   async created () {
-    await this.$store.dispatch('project/getProjectList')
+    await this.$store.dispatch('project/getProjectList', this.filter)
+    await this.$store.dispatch('project/getProjectTags')
   }
 }
 </script>
 
 <style scoped>
   .action-wrapper {
-    text-align: right;
+    display: flex;
+    justify-content: space-between;
     padding-bottom: 10px;
     border-bottom: 1px solid #EBEEF5;
   }

From 2243f4d700c928a10e009633e0da3577087c5f72 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 14:14:59 +0800
Subject: [PATCH 09/14] optimized project

---
 frontend/src/i18n/zh.js                    |  4 ++++
 frontend/src/store/modules/project.js      | 18 +++++++++++++++---
 frontend/src/views/project/ProjectList.vue | 22 +++++++++++++++++++---
 3 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 6b3bcdfa..9ad98d2f 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -221,6 +221,10 @@ export default {
 
   // 项目
   'All Tags': '全部标签',
+  'Project Name': '项目名称',
+  'Project Description': '项目描述',
+  'Tags': '标签',
+  'Enter Tags': '输入标签',
 
   // 定时任务
   'Schedule Name': '定时任务名称',
diff --git a/frontend/src/store/modules/project.js b/frontend/src/store/modules/project.js
index bed02795..eafe9dbb 100644
--- a/frontend/src/store/modules/project.js
+++ b/frontend/src/store/modules/project.js
@@ -2,7 +2,8 @@ import request from '../../api/request'
 
 const state = {
   projectForm: {},
-  projectList: []
+  projectList: [],
+  projectTags: []
 }
 
 const getters = {}
@@ -13,12 +14,15 @@ const mutations = {
   },
   SET_PROJECT_LIST: (state, value) => {
     state.projectList = value
+  },
+  SET_PROJECT_TAGS: (state, value) => {
+    state.projectTags = value
   }
 }
 
 const actions = {
-  getProjectList ({ state, commit }) {
-    request.get('/projects')
+  getProjectList ({ state, commit }, payload) {
+    request.get('/projects', payload)
       .then(response => {
         if (response.data.data) {
           commit('SET_PROJECT_LIST', response.data.data.map(d => {
@@ -28,6 +32,14 @@ const actions = {
         }
       })
   },
+  getProjectTags ({ state, commit }) {
+    request.get('/projects/tags')
+      .then(response => {
+        if (response.data.data) {
+          commit('SET_PROJECT_TAGS', response.data.data.map(d => d.tag))
+        }
+      })
+  },
   addProject ({ state }) {
     request.put('/projects', state.projectForm)
   },
diff --git a/frontend/src/views/project/ProjectList.vue b/frontend/src/views/project/ProjectList.vue
index e6a11af0..5d03de2c 100644
--- a/frontend/src/views/project/ProjectList.vue
+++ b/frontend/src/views/project/ProjectList.vue
@@ -11,7 +11,7 @@
                :inline-message="true"
                ref="projectForm"
                label-position="right">
-        <el-form-item :label="$t('Proejct Name')" prop="name" required>
+        <el-form-item :label="$t('Project Name')" prop="name" required>
           <el-input id="name" v-model="projectForm.name" :placeholder="$t('Project Name')"></el-input>
         </el-form-item>
         <el-form-item :label="$t('Project Description')" prop="description">
@@ -84,8 +84,8 @@
           <el-card
             class="item-card"
           >
-            <i class="btn-edit fa fa-edit" @click="onEdit(item)"></i>
-            <i class="btn-close fa fa-trash-o" @click="onRemove(item)"></i>
+            <i v-if="!isNoProject(item)" class="btn-edit fa fa-edit" @click="onEdit(item)"></i>
+            <i v-if="!isNoProject(item)" class="btn-close fa fa-trash-o" @click="onRemove(item)"></i>
             <el-row>
               <h4 class="title">{{ item.name }}</h4>
             </el-row>
@@ -129,6 +129,7 @@ export default {
     return {
       defaultTags: [],
       dialogVisible: false,
+      isClickAction: false,
       filter: {
         tag: ''
       }
@@ -185,12 +186,22 @@ export default {
       this.$st.sendEv('项目', '提交项目')
     },
     onEdit (row) {
+      this.isClickAction = true
+      setTimeout(() => {
+        this.isClickAction = false
+      }, 100)
+
       this.$store.commit('project/SET_PROJECT_FORM', row)
       this.dialogVisible = true
       this.isEdit = true
       this.$st.sendEv('项目', '修改项目')
     },
     onRemove (row) {
+      this.isClickAction = true
+      setTimeout(() => {
+        this.isClickAction = false
+      }, 100)
+
       this.$confirm(this.$t('Are you sure to delete the project?'), this.$t('Notification'), {
         confirmButtonText: this.$t('Confirm'),
         cancelButtonText: this.$t('Cancel'),
@@ -208,12 +219,17 @@ export default {
       this.$st.sendEv('项目', '删除项目')
     },
     onView (row) {
+      if (this.isClickAction) return
+
       this.$router.push({
         name: 'SpiderList',
         params: {
           project_id: row._id
         }
       })
+    },
+    isNoProject (row) {
+      return row._id === '000000000000000000000000'
     }
   },
   async created () {

From 3f5685cd3195a7c0a0b6a56ea29d0960c4bff975 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 15:48:19 +0800
Subject: [PATCH 10/14] optimized project

---
 frontend/src/i18n/zh.js                    | 1 +
 frontend/src/views/project/ProjectList.vue | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 9ad98d2f..8a3fb800 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -225,6 +225,7 @@ export default {
   'Project Description': '项目描述',
   'Tags': '标签',
   'Enter Tags': '输入标签',
+  'No Project': '无项目',
 
   // 定时任务
   'Schedule Name': '定时任务名称',
diff --git a/frontend/src/views/project/ProjectList.vue b/frontend/src/views/project/ProjectList.vue
index 5d03de2c..b282c82d 100644
--- a/frontend/src/views/project/ProjectList.vue
+++ b/frontend/src/views/project/ProjectList.vue
@@ -77,7 +77,7 @@
       <ul v-else class="list">
         <li
           class="item"
-          v-for="item in projectList"
+          v-for="(item, index) in projectList"
           :key="item._id"
           @click="onView(item)"
         >
@@ -87,7 +87,8 @@
             <i v-if="!isNoProject(item)" class="btn-edit fa fa-edit" @click="onEdit(item)"></i>
             <i v-if="!isNoProject(item)" class="btn-close fa fa-trash-o" @click="onRemove(item)"></i>
             <el-row>
-              <h4 class="title">{{ item.name }}</h4>
+              <h4 v-if="index !== projectList.length - 1" class="title">{{ item.name }}</h4>
+              <h4 v-else class="title">{{ $t('No Project') }}</h4>
             </el-row>
             <el-row>
               <div class="spider-count">

From af689b916a434085dc814700bc9181e30e5b1163 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 16:19:45 +0800
Subject: [PATCH 11/14] optimized project

---
 frontend/src/components/InfoView/SpiderInfoView.vue |  3 +--
 frontend/src/store/modules/project.js               | 10 +++++-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue
index afec2413..48b70a12 100644
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -24,7 +24,6 @@
             :placeholder="$t('Project')"
             filterable
           >
-            <el-option value="000000000000000000000000" :label="$t('No project')"/>
             <el-option
               v-for="p in projectList"
               :key="p._id"
@@ -205,7 +204,7 @@ export default {
 
     // 兼容项目ID
     if (!this.spiderForm.project_id) {
-      this.spiderForm.project_id = '000000000000000000000000'
+      this.$set(this.spiderForm, 'project_id', '000000000000000000000000')
     }
   }
 }
diff --git a/frontend/src/store/modules/project.js b/frontend/src/store/modules/project.js
index eafe9dbb..0c6b504f 100644
--- a/frontend/src/store/modules/project.js
+++ b/frontend/src/store/modules/project.js
@@ -22,7 +22,7 @@ const mutations = {
 
 const actions = {
   getProjectList ({ state, commit }, payload) {
-    request.get('/projects', payload)
+    return request.get('/projects', payload)
       .then(response => {
         if (response.data.data) {
           commit('SET_PROJECT_LIST', response.data.data.map(d => {
@@ -33,7 +33,7 @@ const actions = {
       })
   },
   getProjectTags ({ state, commit }) {
-    request.get('/projects/tags')
+    return request.get('/projects/tags')
       .then(response => {
         if (response.data.data) {
           commit('SET_PROJECT_TAGS', response.data.data.map(d => d.tag))
@@ -41,13 +41,13 @@ const actions = {
       })
   },
   addProject ({ state }) {
-    request.put('/projects', state.projectForm)
+    return request.put('/projects', state.projectForm)
   },
   editProject ({ state }, id) {
-    request.post(`/projects/${id}`, state.projectForm)
+    return request.post(`/projects/${id}`, state.projectForm)
   },
   removeProject ({ state }, id) {
-    request.delete(`/projects/${id}`)
+    return request.delete(`/projects/${id}`)
   }
 }
 

From e063833f1210afe263557a1d79ba1ae81fe56514 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 22:30:34 +0800
Subject: [PATCH 12/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=BB=E5=8A=A1?=
 =?UTF-8?q?=E6=9B=B4=E6=96=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/views/task/TaskDetail.vue | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue
index f52597ca..39aca864 100644
--- a/frontend/src/views/task/TaskDetail.vue
+++ b/frontend/src/views/task/TaskDetail.vue
@@ -137,6 +137,7 @@ export default {
   },
   computed: {
     ...mapState('task', [
+      'taskForm',
       'taskResultsData',
       'taskResultsTotalCount'
     ]),
@@ -164,6 +165,9 @@ export default {
       set (value) {
         this.$store.commit('task/SET_RESULTS_PAGE_SIZE', value)
       }
+    },
+    isRunning () {
+      return ['pending', 'running'].includes(this.taskForm.status)
     }
   },
   methods: {
@@ -197,6 +201,9 @@ export default {
 
     this.getTaskLog()
     this.handle = setInterval(() => {
+      if (!this.isRunning) return
+      this.$store.dispatch('task/getTaskData', this.$route.params.id)
+      this.$store.dispatch('task/getTaskResults', this.$route.params.id)
       this.getTaskLog()
     }, 5000)
   },

From 5740774ccc7b3d5557c4e9af5b0a933016a0c272 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sun, 2 Feb 2020 22:56:11 +0800
Subject: [PATCH 13/14] =?UTF-8?q?=E6=B7=BB=E5=8A=A0demo=E7=88=AC=E8=99=AB?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/entity/config_spider.go            | 10 +++
 backend/main.go                            | 35 +++++-----
 backend/model/spider.go                    |  2 +-
 backend/routes/projects.go                 | 18 +++++
 backend/services/spider.go                 | 78 ++++++++++++++++++++++
 spiders/chinaz/Spiderfile                  |  4 ++
 spiders/chinaz/chinaz/pipelines.py         | 21 ------
 spiders/chinaz/chinaz/settings.py          |  2 +-
 spiders/realestate/realestate/pipelines.py | 19 ------
 spiders/realestate/realestate/settings.py  |  2 +-
 10 files changed, 130 insertions(+), 61 deletions(-)
 create mode 100644 spiders/chinaz/Spiderfile

diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go
index d9e085d2..bb1295e7 100644
--- a/backend/entity/config_spider.go
+++ b/backend/entity/config_spider.go
@@ -1,12 +1,22 @@
 package entity
 
 type ConfigSpiderData struct {
+	// 通用
+	Name        string `yaml:"name" json:"name"`
+	DisplayName string `yaml:"display_name" json:"display_name"`
+	Col         string `yaml:"col" json:"col"`
+	Remark      string `yaml:"remark" json:"remark"`
+
+	// 可配置爬虫
 	Version    string            `yaml:"version" json:"version"`
 	Engine     string            `yaml:"engine" json:"engine"`
 	StartUrl   string            `yaml:"start_url" json:"start_url"`
 	StartStage string            `yaml:"start_stage" json:"start_stage"`
 	Stages     []Stage           `yaml:"stages" json:"stages"`
 	Settings   map[string]string `yaml:"settings" json:"settings"`
+
+	// 自定义爬虫
+	Cmd string `yaml:"cmd" json:"cmd"`
 }
 
 type Stage struct {
diff --git a/backend/main.go b/backend/main.go
index d9d44a12..ab0d0e7b 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -39,7 +39,6 @@ func main() {
 		log.SetLevelFromString(logLevel)
 	}
 	log.Info("initialized log config successfully")
-
 	if viper.GetString("log.isDeletePeriodically") == "Y" {
 		err := services.InitDeleteLogPeriodically()
 		if err != nil {
@@ -74,8 +73,24 @@ func main() {
 			debug.PrintStack()
 			panic(err)
 		}
+		log.Info("initialized schedule successfully")
+
+		// 初始化用户服务
+		if err := services.InitUserService(); err != nil {
+			log.Error("init user service error:" + err.Error())
+			debug.PrintStack()
+			panic(err)
+		}
+		log.Info("initialized user service successfully")
+
+		// 初始化依赖服务
+		if err := services.InitDepsFetcher(); err != nil {
+			log.Error("init dependency fetcher error:" + err.Error())
+			debug.PrintStack()
+			panic(err)
+		}
+		log.Info("initialized dependency fetcher successfully")
 	}
-	log.Info("initialized schedule successfully")
 
 	// 初始化任务执行器
 	if err := services.InitTaskExecutor(); err != nil {
@@ -100,22 +115,6 @@ func main() {
 	}
 	log.Info("initialized spider service successfully")
 
-	// 初始化用户服务
-	if err := services.InitUserService(); err != nil {
-		log.Error("init user service error:" + err.Error())
-		debug.PrintStack()
-		panic(err)
-	}
-	log.Info("initialized user service successfully")
-
-	// 初始化依赖服务
-	if err := services.InitDepsFetcher(); err != nil {
-		log.Error("init dependency fetcher error:" + err.Error())
-		debug.PrintStack()
-		panic(err)
-	}
-	log.Info("initialized dependency fetcher successfully")
-
 	// 初始化RPC服务
 	if err := services.InitRpcService(); err != nil {
 		log.Error("init rpc service error:" + err.Error())
diff --git a/backend/model/spider.go b/backend/model/spider.go
index 475b12e2..2baeb6ed 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -168,7 +168,7 @@ func GetSpiderByName(name string) Spider {
 	defer s.Close()
 
 	var result Spider
-	if err := c.Find(bson.M{"name": name}).One(&result); err != nil {
+	if err := c.Find(bson.M{"name": name}).One(&result); err != nil && err != mgo.ErrNotFound {
 		log.Errorf("get spider error: %s, spider_name: %s", err.Error(), name)
 		//debug.PrintStack()
 		return result
diff --git a/backend/routes/projects.go b/backend/routes/projects.go
index b1f99ab3..34b2d7f4 100644
--- a/backend/routes/projects.go
+++ b/backend/routes/projects.go
@@ -123,6 +123,24 @@ func DeleteProject(c *gin.Context) {
 		return
 	}
 
+	// 获取相关的爬虫
+	var spiders []model.Spider
+	s, col := database.GetCol("spiders")
+	defer s.Close()
+	if err := col.Find(bson.M{"project_id": bson.ObjectIdHex(id)}).All(&spiders); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 将爬虫的项目ID置空
+	for _, spider := range spiders {
+		spider.ProjectId = bson.ObjectIdHex(constants.ObjectIdNull)
+		if err := spider.Save(); err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+	}
+
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
diff --git a/backend/services/spider.go b/backend/services/spider.go
index a03d4dc8..b395a956 100644
--- a/backend/services/spider.go
+++ b/backend/services/spider.go
@@ -14,7 +14,10 @@ import (
 	"github.com/globalsign/mgo/bson"
 	"github.com/satori/go.uuid"
 	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime/debug"
 )
@@ -264,5 +267,80 @@ func InitSpiderService() error {
 	// 启动定时任务
 	c.Start()
 
+	if model.IsMaster() {
+		// 添加Demo爬虫
+		templateSpidersDir := "../spiders"
+		for _, info := range utils.ListDir(templateSpidersDir) {
+			if !info.IsDir() {
+				continue
+			}
+			spiderName := info.Name()
+
+			// 如果爬虫在数据库中不存在，则添加
+			spider := model.GetSpiderByName(spiderName)
+			if spider.Name != "" {
+				// 存在同名爬虫，跳过
+				continue
+			}
+
+			// 拷贝爬虫
+			templateSpiderPath := path.Join(templateSpidersDir, spiderName)
+			spiderPath := path.Join(viper.GetString("spider.path"), spiderName)
+			if utils.Exists(spiderPath) {
+				utils.RemoveFiles(spiderPath)
+			}
+			if err := utils.CopyDir(templateSpiderPath, spiderPath); err != nil {
+				log.Errorf("copy error: " + err.Error())
+				debug.PrintStack()
+				continue
+			}
+
+			// 构造配置数据
+			configData := entity.ConfigSpiderData{}
+
+			// 读取YAML文件
+			yamlFile, err := ioutil.ReadFile(path.Join(spiderPath, "Spiderfile"))
+			if err != nil {
+				log.Errorf("read yaml error: " + err.Error())
+				//debug.PrintStack()
+				continue
+			}
+
+			// 反序列化
+			if err := yaml.Unmarshal(yamlFile, &configData); err != nil {
+				log.Errorf("unmarshal error: " + err.Error())
+				debug.PrintStack()
+				continue
+			}
+
+			// 添加该爬虫到数据库
+			spider = model.Spider{
+				Id:          bson.NewObjectId(),
+				Name:        configData.Name,
+				DisplayName: configData.DisplayName,
+				Type:        constants.Customized,
+				Col:         configData.Col,
+				Cmd:         configData.Cmd,
+				Src:         spiderPath,
+				Remark:      configData.Remark,
+				ProjectId:   bson.ObjectIdHex(constants.ObjectIdNull),
+				FileId:      bson.ObjectIdHex(constants.ObjectIdNull),
+			}
+			if err := spider.Add(); err != nil {
+				log.Errorf("add spider error: " + err.Error())
+				debug.PrintStack()
+				continue
+			}
+
+			// 上传爬虫到GridFS
+			if err := UploadSpiderToGridFsFromMaster(spider); err != nil {
+				log.Errorf("upload spider error: " + err.Error())
+				debug.PrintStack()
+				continue
+			}
+		}
+
+	}
+
 	return nil
 }
diff --git a/spiders/chinaz/Spiderfile b/spiders/chinaz/Spiderfile
new file mode 100644
index 00000000..d36c7cf2
--- /dev/null
+++ b/spiders/chinaz/Spiderfile
@@ -0,0 +1,4 @@
+name: "chinaz"
+display_name: "站长之家 (Scrapy)"
+col: "results_chinaz"
+cmd: "scrapy crawl chinaz_spider"
\ No newline at end of file
diff --git a/spiders/chinaz/chinaz/pipelines.py b/spiders/chinaz/chinaz/pipelines.py
index 747de355..b29f9eb7 100644
--- a/spiders/chinaz/chinaz/pipelines.py
+++ b/spiders/chinaz/chinaz/pipelines.py
@@ -5,24 +5,3 @@
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 
-import os
-
-from pymongo import MongoClient
-
-MONGO_HOST = os.environ.get('MONGO_HOST') or 'localhost'
-MONGO_PORT = int(os.environ.get('MONGO_PORT') or '27017')
-MONGO_DB = os.environ.get('MONGO_DB') or 'crawlab_test'
-
-
-class MongoPipeline(object):
-    mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
-    db = mongo[MONGO_DB]
-    col_name = os.environ.get('CRAWLAB_COLLECTION') or 'sites'
-    col = db[col_name]
-
-    def process_item(self, item, spider):
-        item['task_id'] = os.environ.get('CRAWLAB_TASK_ID')
-        item['_id'] = item['domain']
-        if self.col.find_one({'_id': item['_id']}) is None:
-            self.col.save(item)
-        return item
diff --git a/spiders/chinaz/chinaz/settings.py b/spiders/chinaz/chinaz/settings.py
index 41fb31bf..932ec9ed 100644
--- a/spiders/chinaz/chinaz/settings.py
+++ b/spiders/chinaz/chinaz/settings.py
@@ -65,7 +65,7 @@ ROBOTSTXT_OBEY = True
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
-   'chinaz.pipelines.MongoPipeline': 300,
+   'crawlab.pipelines.CrawlabMongoPipeline': 300,
 }
 
 # Enable and configure the AutoThrottle extension (disabled by default)
diff --git a/spiders/realestate/realestate/pipelines.py b/spiders/realestate/realestate/pipelines.py
index 7b9eb9f2..3371792b 100644
--- a/spiders/realestate/realestate/pipelines.py
+++ b/spiders/realestate/realestate/pipelines.py
@@ -4,22 +4,3 @@
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
-import os
-
-from pymongo import MongoClient
-
-MONGO_HOST = os.environ.get('MONGO_HOST') or 'localhost'
-MONGO_PORT = int(os.environ.get('MONGO_PORT') or '27017')
-MONGO_DB = os.environ.get('MONGO_DB') or 'crawlab_test'
-
-
-class MongoPipeline(object):
-    mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
-    db = mongo[MONGO_DB]
-    col_name = os.environ.get('CRAWLAB_COLLECTION')
-    col = db[col_name]
-
-    def process_item(self, item, spider):
-        item['task_id'] = os.environ.get('CRAWLAB_TASK_ID')
-        self.col.save(item)
-        return item
diff --git a/spiders/realestate/realestate/settings.py b/spiders/realestate/realestate/settings.py
index da1ada29..758f8ed0 100644
--- a/spiders/realestate/realestate/settings.py
+++ b/spiders/realestate/realestate/settings.py
@@ -64,7 +64,7 @@ ROBOTSTXT_OBEY = True
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
-    'realestate.pipelines.MongoPipeline': 300,
+    'crawlab.pipelines.CrawlabMongoPipeline': 300,
 }
 
 # Enable and configure the AutoThrottle extension (disabled by default)

From cb6a8f79d898579ed6b09e77cd7192323d268b84 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Mon, 3 Feb 2020 09:21:41 +0800
Subject: [PATCH 14/14] added demo spiders

---
 backend/entity/config_spider.go          |  2 +-
 backend/services/config_spider.go        | 12 ++++
 backend/services/spider.go               | 74 ++++++++++++++++--------
 spiders/amazon_config/Spiderfile         | 51 ++++++++++++++++
 spiders/autohome_config/Spiderfile       | 57 ++++++++++++++++++
 spiders/baidu_config/Spiderfile          | 39 +++++++++++++
 spiders/chinaz/Spiderfile                |  1 +
 spiders/csdn_config/Spiderfile           | 60 +++++++++++++++++++
 spiders/douban_config/Spiderfile         | 57 ++++++++++++++++++
 spiders/jd/Spiderfile                    |  5 ++
 spiders/jd/jd/items.py                   |  1 +
 spiders/jd/jd/pipelines.py               | 11 ----
 spiders/jd/jd/settings.py                |  4 +-
 spiders/jd/jd/spiders/jd_spider.py       | 14 ++++-
 spiders/realestate/Spiderfile            |  4 ++
 spiders/sinastock/Spiderfile             |  5 ++
 spiders/sinastock/sinastock/pipelines.py | 22 -------
 spiders/sinastock/sinastock/settings.py  |  2 +-
 spiders/v2ex_config/Spiderfile           | 54 +++++++++++++++++
 spiders/xueqiu/Spiderfile                |  5 ++
 spiders/xueqiu/xueqiu/pipelines.py       | 23 --------
 spiders/xueqiu/xueqiu/settings.py        |  4 +-
 spiders/xueqiu_config/Spiderfile         | 39 +++++++++++++
 spiders/zongheng_config/Spiderfile       | 45 ++++++++++++++
 24 files changed, 504 insertions(+), 87 deletions(-)
 create mode 100644 spiders/amazon_config/Spiderfile
 create mode 100644 spiders/autohome_config/Spiderfile
 create mode 100644 spiders/baidu_config/Spiderfile
 create mode 100644 spiders/csdn_config/Spiderfile
 create mode 100644 spiders/douban_config/Spiderfile
 create mode 100644 spiders/jd/Spiderfile
 create mode 100644 spiders/realestate/Spiderfile
 create mode 100644 spiders/sinastock/Spiderfile
 create mode 100644 spiders/v2ex_config/Spiderfile
 create mode 100644 spiders/xueqiu/Spiderfile
 create mode 100644 spiders/xueqiu_config/Spiderfile
 create mode 100644 spiders/zongheng_config/Spiderfile

diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go
index bb1295e7..054ee2fe 100644
--- a/backend/entity/config_spider.go
+++ b/backend/entity/config_spider.go
@@ -6,9 +6,9 @@ type ConfigSpiderData struct {
 	DisplayName string `yaml:"display_name" json:"display_name"`
 	Col         string `yaml:"col" json:"col"`
 	Remark      string `yaml:"remark" json:"remark"`
+	Type        string `yaml:"type" bson:"type"`
 
 	// 可配置爬虫
-	Version    string            `yaml:"version" json:"version"`
 	Engine     string            `yaml:"engine" json:"engine"`
 	StartUrl   string            `yaml:"start_url" json:"start_url"`
 	StartStage string            `yaml:"start_stage" json:"start_stage"`
diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go
index fe0a3da1..29e1c2ca 100644
--- a/backend/services/config_spider.go
+++ b/backend/services/config_spider.go
@@ -6,6 +6,7 @@ import (
 	"crawlab/entity"
 	"crawlab/model"
 	"crawlab/model/config_spider"
+	"crawlab/services/spider_handler"
 	"crawlab/utils"
 	"errors"
 	"fmt"
@@ -227,6 +228,17 @@ func ProcessSpiderFilesFromConfigData(spider model.Spider, configData entity.Con
 	spider.FileId = fid
 	_ = spider.Save()
 
+	// 获取爬虫同步实例
+	spiderSync := spider_handler.SpiderSync{
+		Spider: spider,
+	}
+
+	// 获取gfFile
+	gfFile2 := model.GetGridFs(spider.FileId)
+
+	// 生成MD5
+	spiderSync.CreateMd5File(gfFile2.Md5)
+
 	return nil
 }
 
diff --git a/backend/services/spider.go b/backend/services/spider.go
index b395a956..48777042 100644
--- a/backend/services/spider.go
+++ b/backend/services/spider.go
@@ -313,30 +313,58 @@ func InitSpiderService() error {
 				continue
 			}
 
-			// 添加该爬虫到数据库
-			spider = model.Spider{
-				Id:          bson.NewObjectId(),
-				Name:        configData.Name,
-				DisplayName: configData.DisplayName,
-				Type:        constants.Customized,
-				Col:         configData.Col,
-				Cmd:         configData.Cmd,
-				Src:         spiderPath,
-				Remark:      configData.Remark,
-				ProjectId:   bson.ObjectIdHex(constants.ObjectIdNull),
-				FileId:      bson.ObjectIdHex(constants.ObjectIdNull),
-			}
-			if err := spider.Add(); err != nil {
-				log.Errorf("add spider error: " + err.Error())
-				debug.PrintStack()
-				continue
-			}
+			if configData.Type == constants.Customized {
+				// 添加该爬虫到数据库
+				spider = model.Spider{
+					Id:          bson.NewObjectId(),
+					Name:        configData.Name,
+					DisplayName: configData.DisplayName,
+					Type:        constants.Customized,
+					Col:         configData.Col,
+					Src:         spiderPath,
+					Remark:      configData.Remark,
+					ProjectId:   bson.ObjectIdHex(constants.ObjectIdNull),
+					FileId:      bson.ObjectIdHex(constants.ObjectIdNull),
+					Cmd:         configData.Cmd,
+				}
+				if err := spider.Add(); err != nil {
+					log.Errorf("add spider error: " + err.Error())
+					debug.PrintStack()
+					continue
+				}
 
-			// 上传爬虫到GridFS
-			if err := UploadSpiderToGridFsFromMaster(spider); err != nil {
-				log.Errorf("upload spider error: " + err.Error())
-				debug.PrintStack()
-				continue
+				// 上传爬虫到GridFS
+				if err := UploadSpiderToGridFsFromMaster(spider); err != nil {
+					log.Errorf("upload spider error: " + err.Error())
+					debug.PrintStack()
+					continue
+				}
+			} else if configData.Type == constants.Configurable || configData.Type == "config" {
+				// 添加该爬虫到数据库
+				spider = model.Spider{
+					Id:          bson.NewObjectId(),
+					Name:        configData.Name,
+					DisplayName: configData.DisplayName,
+					Type:        constants.Configurable,
+					Col:         configData.Col,
+					Src:         spiderPath,
+					Remark:      configData.Remark,
+					ProjectId:   bson.ObjectIdHex(constants.ObjectIdNull),
+					FileId:      bson.ObjectIdHex(constants.ObjectIdNull),
+					Config:      configData,
+				}
+				if err := spider.Add(); err != nil {
+					log.Errorf("add spider error: " + err.Error())
+					debug.PrintStack()
+					continue
+				}
+
+				// 根据序列化后的数据处理爬虫文件
+				if err := ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
+					log.Errorf("add spider error: " + err.Error())
+					debug.PrintStack()
+					continue
+				}
 			}
 		}
 
diff --git a/spiders/amazon_config/Spiderfile b/spiders/amazon_config/Spiderfile
new file mode 100644
index 00000000..eea8a538
--- /dev/null
+++ b/spiders/amazon_config/Spiderfile
@@ -0,0 +1,51 @@
+name: "amazon_config"
+display_name: "亚马逊中国（可配置）"
+remark: "亚马逊中国搜索手机，列表+分页"
+type: "configurable"
+col: "results_amazon_config"
+engine: scrapy
+start_url: https://www.amazon.cn/s?k=%E6%89%8B%E6%9C%BA&__mk_zh_CN=%E4%BA%9A%E9%A9%AC%E9%80%8A%E7%BD%91%E7%AB%99&ref=nb_sb_noss_2
+start_stage: list
+stages:
+- name: list
+  is_list: true
+  list_css: .s-result-item
+  list_xpath: ""
+  page_css: .a-last > a
+  page_xpath: ""
+  page_attr: href
+  fields:
+  - name: title
+    css: span.a-text-normal
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: url
+    css: .a-link-normal
+    xpath: ""
+    attr: href
+    next_stage: ""
+    remark: ""
+  - name: price
+    css: ""
+    xpath: .//*[@class="a-price-whole"]
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: price_fraction
+    css: ""
+    xpath: .//*[@class="a-price-fraction"]
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: img
+    css: .s-image-square-aspect > img
+    xpath: ""
+    attr: src
+    next_stage: ""
+    remark: ""
+settings:
+  ROBOTSTXT_OBEY: "false"
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML,
+    like Gecko) Chrome/78.0.3904.108 Safari/537.36
diff --git a/spiders/autohome_config/Spiderfile b/spiders/autohome_config/Spiderfile
new file mode 100644
index 00000000..e69880cb
--- /dev/null
+++ b/spiders/autohome_config/Spiderfile
@@ -0,0 +1,57 @@
+name: "autohome_config"
+display_name: "汽车之家（可配置）"
+remark: "汽车之家文章，列表+详情+分页"
+type: "configurable"
+col: "results_autohome_config"
+engine: scrapy
+start_url: https://www.autohome.com.cn/all/
+start_stage: list
+stages:
+- name: list
+  is_list: true
+  list_css: ul.article > li
+  list_xpath: ""
+  page_css: a.page-item-next
+  page_xpath: ""
+  page_attr: href
+  fields:
+  - name: title
+    css: li > a > h3
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: url
+    css: li > a
+    xpath: ""
+    attr: href
+    next_stage: ""
+    remark: ""
+  - name: abstract
+    css: li > a > p
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: time
+    css: li > a .fn-left
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: views
+    css: li > a .fn-right > em:first-child
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: comments
+    css: li > a .fn-right > em:last-child
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+settings:
+  ROBOTSTXT_OBEY: "false"
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML,
+    like Gecko) Chrome/78.0.3904.108 Safari/537.36
diff --git a/spiders/baidu_config/Spiderfile b/spiders/baidu_config/Spiderfile
new file mode 100644
index 00000000..5266b85b
--- /dev/null
+++ b/spiders/baidu_config/Spiderfile
@@ -0,0 +1,39 @@
+name: "baidu_config"
+display_name: "百度搜索（可配置）"
+remark: "百度搜索Crawlab，列表+分页"
+type: "configurable"
+col: "results_baidu_config"
+engine: scrapy
+start_url: http://www.baidu.com/s?wd=crawlab
+start_stage: list
+stages:
+- name: list
+  is_list: true
+  list_css: ""
+  list_xpath: //body
+  page_css: ""
+  page_xpath: //body
+  page_attr: href
+  fields:
+  - name: title
+    css: ""
+    xpath: .//h3/a
+    attr: href
+    next_stage: ""
+    remark: ""
+  - name: url
+    css: ""
+    xpath: .//h3/a
+    attr: href
+    next_stage: ""
+    remark: ""
+  - name: abstract
+    css: ""
+    xpath: .//*[@class="c-abstract"]
+    attr: href
+    next_stage: ""
+    remark: ""
+settings:
+  ROBOTSTXT_OBEY: "false"
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML,
+    like Gecko) Chrome/78.0.3904.108 Safari/537.36
diff --git a/spiders/chinaz/Spiderfile b/spiders/chinaz/Spiderfile
index d36c7cf2..2fb940bb 100644
--- a/spiders/chinaz/Spiderfile
+++ b/spiders/chinaz/Spiderfile
@@ -1,4 +1,5 @@
 name: "chinaz"
 display_name: "站长之家 (Scrapy)"
 col: "results_chinaz"
+type: "customized"
 cmd: "scrapy crawl chinaz_spider"
\ No newline at end of file
diff --git a/spiders/csdn_config/Spiderfile b/spiders/csdn_config/Spiderfile
new file mode 100644
index 00000000..67f4f8c5
--- /dev/null
+++ b/spiders/csdn_config/Spiderfile
@@ -0,0 +1,60 @@
+name: "csdn_config"
+display_name: "CSDN（可配置）"
+remark: "CSDN Crawlab 文章，列表+详情+分页"
+type: "configurable"
+col: "results_csdn_config"
+engine: scrapy
+start_url: https://so.csdn.net/so/search/s.do?q=crawlab
+start_stage: list
+stages:
+- name: list
+  is_list: true
+  list_css: .search-list-con > .search-list
+  list_xpath: ""
+  page_css: a.btn-next
+  page_xpath: ""
+  page_attr: href
+  fields:
+  - name: url
+    css: ""
+    xpath: .//*[@class="limit_width"]/a
+    attr: href
+    next_stage: detail
+    remark: ""
+- name: detail
+  is_list: false
+  list_css: ""
+  list_xpath: ""
+  page_css: ""
+  page_xpath: ""
+  page_attr: ""
+  fields:
+  - name: content
+    css: ""
+    xpath: .//div[@id="content_views"]
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: views
+    css: .read-count
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: title
+    css: .title-article
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: author
+    css: .follow-nickName
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+settings:
+  AUTOTHROTTLE_ENABLED: "false"
+  ROBOTSTXT_OBEY: "false"
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML,
+    like Gecko) Chrome/79.0.3945.117 Safari/537.36
diff --git a/spiders/douban_config/Spiderfile b/spiders/douban_config/Spiderfile
new file mode 100644
index 00000000..84f0647a
--- /dev/null
+++ b/spiders/douban_config/Spiderfile
@@ -0,0 +1,57 @@
+name: "douban_config"
+display_name: "豆瓣读书（可配置）"
+remark: "豆瓣读书新书推荐，列表"
+type: "configurable"
+col: "results_douban_config"
+engine: scrapy
+start_url: https://book.douban.com/latest
+start_stage: list
+stages:
+- name: list
+  is_list: true
+  list_css: ul.cover-col-4 > li
+  list_xpath: ""
+  page_css: ""
+  page_xpath: ""
+  page_attr: ""
+  fields:
+  - name: title
+    css: h2 > a
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: url
+    css: h2 > a
+    xpath: ""
+    attr: href
+    next_stage: ""
+    remark: ""
+  - name: img
+    css: a.cover img
+    xpath: ""
+    attr: src
+    next_stage: ""
+    remark: ""
+  - name: rating
+    css: p.rating > .color-lightgray
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: abstract
+    css: p:last-child
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: info
+    css: .color-gray
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+settings:
+  ROBOTSTXT_OBEY: "false"
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML,
+    like Gecko) Chrome/78.0.3904.108 Safari/537.36
diff --git a/spiders/jd/Spiderfile b/spiders/jd/Spiderfile
new file mode 100644
index 00000000..d090472b
--- /dev/null
+++ b/spiders/jd/Spiderfile
@@ -0,0 +1,5 @@
+name: "jd"
+display_name: "京东 (Scrapy)"
+col: "results_jd"
+type: "customized"
+cmd: "scrapy crawl jd_spider"
\ No newline at end of file
diff --git a/spiders/jd/jd/items.py b/spiders/jd/jd/items.py
index 9a7ba1cb..b2c5e647 100644
--- a/spiders/jd/jd/items.py
+++ b/spiders/jd/jd/items.py
@@ -12,3 +12,4 @@ class JdItem(scrapy.Item):
     # define the fields for your item here like:
     name = scrapy.Field()
     price = scrapy.Field()
+    url = scrapy.Field()
diff --git a/spiders/jd/jd/pipelines.py b/spiders/jd/jd/pipelines.py
index b862b7e7..5a7d7cbf 100644
--- a/spiders/jd/jd/pipelines.py
+++ b/spiders/jd/jd/pipelines.py
@@ -4,14 +4,3 @@
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
-from pymongo import MongoClient
-
-
-class JdPipeline(object):
-    mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
-    db = mongo[MONGO_DB]
-    col_name = os.environ.get('CRAWLAB_COLLECTION') or 'jd_products'
-    col = db[col_name]
-
-    def process_item(self, item, spider):
-        return item
diff --git a/spiders/jd/jd/settings.py b/spiders/jd/jd/settings.py
index d83206b2..ef89ed0c 100644
--- a/spiders/jd/jd/settings.py
+++ b/spiders/jd/jd/settings.py
@@ -19,7 +19,7 @@ NEWSPIDER_MODULE = 'jd.spiders'
 #USER_AGENT = 'jd (+http://www.yourdomain.com)'
 
 # Obey robots.txt rules
-ROBOTSTXT_OBEY = True
+ROBOTSTXT_OBEY = False
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
 #CONCURRENT_REQUESTS = 32
@@ -65,7 +65,7 @@ ROBOTSTXT_OBEY = True
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
-   'jd.pipelines.JdPipeline': 300,
+   'crawlab.pipelines.CrawlabMongoPipeline': 300,
 }
 
 # Enable and configure the AutoThrottle extension (disabled by default)
diff --git a/spiders/jd/jd/spiders/jd_spider.py b/spiders/jd/jd/spiders/jd_spider.py
index 01113a7e..4ec94fa9 100644
--- a/spiders/jd/jd/spiders/jd_spider.py
+++ b/spiders/jd/jd/spiders/jd_spider.py
@@ -1,11 +1,21 @@
 # -*- coding: utf-8 -*-
 import scrapy
 
+from jd.items import JdItem
+
 
 class JdSpiderSpider(scrapy.Spider):
     name = 'jd_spider'
     allowed_domains = ['jd.com']
-    start_urls = ['http://jd.com/']
+
+    def start_requests(self):
+    	for i in range(1, 50):
+    		yield scrapy.Request(url=f'https://search.jd.com/Search?keyword=手机&enc=utf-8&page={i}')
 
     def parse(self, response):
-        pass
+        for el in response.css('.gl-item'):
+            yield JdItem(
+                url=el.css('.p-name > a::attr("href")').extract_first(),
+                name=el.css('.p-name > a::attr("title")').extract_first(),
+                price=float(el.css('.p-price i::text').extract_first()),
+            )
diff --git a/spiders/realestate/Spiderfile b/spiders/realestate/Spiderfile
new file mode 100644
index 00000000..772e8312
--- /dev/null
+++ b/spiders/realestate/Spiderfile
@@ -0,0 +1,4 @@
+name: "realestate"
+display_name: "链家网 (Scrapy)"
+col: "results_realestate"
+cmd: "scrapy crawl lianjia"
\ No newline at end of file
diff --git a/spiders/sinastock/Spiderfile b/spiders/sinastock/Spiderfile
new file mode 100644
index 00000000..b110cb48
--- /dev/null
+++ b/spiders/sinastock/Spiderfile
@@ -0,0 +1,5 @@
+name: "sinastock"
+display_name: "新浪股票 (Scrapy)"
+type: "customized"
+col: "results_sinastock"
+cmd: "scrapy crawl sinastock_spider"
\ No newline at end of file
diff --git a/spiders/sinastock/sinastock/pipelines.py b/spiders/sinastock/sinastock/pipelines.py
index e666c50d..5a7d7cbf 100644
--- a/spiders/sinastock/sinastock/pipelines.py
+++ b/spiders/sinastock/sinastock/pipelines.py
@@ -4,25 +4,3 @@
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
-import os
-
-from pymongo import MongoClient
-
-
-class SinastockPipeline(object):
-    mongo = MongoClient(
-        host=os.environ.get('MONGO_HOST') or 'localhost',
-        port=int(os.environ.get('MONGO_PORT') or 27017)
-    )
-    db = mongo[os.environ.get('MONGO_DB') or 'crawlab_test']
-    col = db.get_collection(os.environ.get('CRAWLAB_COLLECTION') or 'stock_news')
-
-    # create indexes
-    col.create_index('stocks')
-    col.create_index('url')
-
-    def process_item(self, item, spider):
-        item['task_id'] = os.environ.get('CRAWLAB_TASK_ID')
-        if self.col.find_one({'url': item['url']}) is None:
-            self.col.save(item)
-            return item
diff --git a/spiders/sinastock/sinastock/settings.py b/spiders/sinastock/sinastock/settings.py
index c63c2eb5..3e01d3ca 100644
--- a/spiders/sinastock/sinastock/settings.py
+++ b/spiders/sinastock/sinastock/settings.py
@@ -64,7 +64,7 @@ ROBOTSTXT_OBEY = True
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
-    'sinastock.pipelines.SinastockPipeline': 300,
+    'crawlab.pipelines.CrawlabMongoPipeline': 300,
 }
 
 # Enable and configure the AutoThrottle extension (disabled by default)
diff --git a/spiders/v2ex_config/Spiderfile b/spiders/v2ex_config/Spiderfile
new file mode 100644
index 00000000..bb18d40a
--- /dev/null
+++ b/spiders/v2ex_config/Spiderfile
@@ -0,0 +1,54 @@
+name: "v2ex_config"
+display_name: "V2ex（可配置）"
+remark: "V2ex，列表+详情"
+type: "configurable"
+col: "results_v2ex_config"
+engine: scrapy
+start_url: https://v2ex.com/
+start_stage: list
+stages:
+- name: list
+  is_list: true
+  list_css: .cell.item
+  list_xpath: ""
+  page_css: ""
+  page_xpath: ""
+  page_attr: href
+  fields:
+  - name: title
+    css: a.topic-link
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: url
+    css: a.topic-link
+    xpath: ""
+    attr: href
+    next_stage: detail
+    remark: ""
+  - name: replies
+    css: .count_livid
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+- name: detail
+  is_list: false
+  list_css: ""
+  list_xpath: ""
+  page_css: ""
+  page_xpath: ""
+  page_attr: ""
+  fields:
+  - name: content
+    css: ""
+    xpath: .//*[@class="markdown_body"]
+    attr: ""
+    next_stage: ""
+    remark: ""
+settings:
+  AUTOTHROTTLE_ENABLED: "true"
+  ROBOTSTXT_OBEY: "false"
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML,
+    like Gecko) Chrome/79.0.3945.117 Safari/537.36
diff --git a/spiders/xueqiu/Spiderfile b/spiders/xueqiu/Spiderfile
new file mode 100644
index 00000000..38aa5dbe
--- /dev/null
+++ b/spiders/xueqiu/Spiderfile
@@ -0,0 +1,5 @@
+name: "xueqiu"
+display_name: "雪球网 (Scrapy)"
+type: "customized"
+col: "results_xueqiu"
+cmd: "scrapy crawl xueqiu_spider"
\ No newline at end of file
diff --git a/spiders/xueqiu/xueqiu/pipelines.py b/spiders/xueqiu/xueqiu/pipelines.py
index 210ce7ac..5a7d7cbf 100644
--- a/spiders/xueqiu/xueqiu/pipelines.py
+++ b/spiders/xueqiu/xueqiu/pipelines.py
@@ -4,26 +4,3 @@
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
-import os
-
-from pymongo import MongoClient
-
-
-class XueqiuPipeline(object):
-    mongo = MongoClient(
-        host=os.environ.get('MONGO_HOST') or 'localhost',
-        port=int(os.environ.get('MONGO_PORT') or 27017)
-    )
-    db = mongo[os.environ.get('MONGO_DB') or 'crawlab_test']
-    col = db.get_collection(os.environ.get('CRAWLAB_COLLECTION') or 'results_xueqiu')
-
-    # create indexes
-    col.create_index('stocks')
-    col.create_index('id')
-    col.create_index('url')
-
-    def process_item(self, item, spider):
-        item['task_id'] = os.environ.get('CRAWLAB_TASK_ID')
-        if self.col.find_one({'id': item['id']}) is None:
-            self.col.save(item)
-            return item
diff --git a/spiders/xueqiu/xueqiu/settings.py b/spiders/xueqiu/xueqiu/settings.py
index b44a74e1..1d898e2f 100644
--- a/spiders/xueqiu/xueqiu/settings.py
+++ b/spiders/xueqiu/xueqiu/settings.py
@@ -18,7 +18,7 @@ NEWSPIDER_MODULE = 'xueqiu.spiders'
 USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
 
 # Obey robots.txt rules
-ROBOTSTXT_OBEY = True
+ROBOTSTXT_OBEY = False
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
 # CONCURRENT_REQUESTS = 32
@@ -64,7 +64,7 @@ ROBOTSTXT_OBEY = True
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
-    'xueqiu.pipelines.XueqiuPipeline': 300,
+    'crawlab.pipelines.CrawlabMongoPipeline': 300,
 }
 
 # Enable and configure the AutoThrottle extension (disabled by default)
diff --git a/spiders/xueqiu_config/Spiderfile b/spiders/xueqiu_config/Spiderfile
new file mode 100644
index 00000000..0de50e9e
--- /dev/null
+++ b/spiders/xueqiu_config/Spiderfile
@@ -0,0 +1,39 @@
+name: "xueqiu_config"
+display_name: "雪球网（可配置）"
+remark: "雪球网新闻，列表"
+type: "configurable"
+col: "results_xueqiu_config"
+engine: scrapy
+start_url: https://xueqiu.com/
+start_stage: list
+stages:
+- name: list
+  is_list: true
+  list_css: ""
+  list_xpath: .//*[contains(@class, "AnonymousHome_home__timeline__item")]
+  page_css: ""
+  page_xpath: ""
+  page_attr: ""
+  fields:
+  - name: title
+    css: h3 > a
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: url
+    css: h3 > a
+    xpath: ""
+    attr: href
+    next_stage: ""
+    remark: ""
+  - name: abstract
+    css: p
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+settings:
+  ROBOTSTXT_OBEY: "false"
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML,
+    like Gecko) Chrome/78.0.3904.108 Safari/537.36
diff --git a/spiders/zongheng_config/Spiderfile b/spiders/zongheng_config/Spiderfile
new file mode 100644
index 00000000..0163fac7
--- /dev/null
+++ b/spiders/zongheng_config/Spiderfile
@@ -0,0 +1,45 @@
+name: "zongheng_config"
+display_name: "纵横（可配置）"
+remark: "纵横小说网，列表"
+type: "configurable"
+col: "results_zongheng_config"
+engine: scrapy
+start_url: http://www.zongheng.com/rank/details.html?rt=1&d=1
+start_stage: list
+stages:
+- name: list
+  is_list: true
+  list_css: .rank_d_list
+  list_xpath: ""
+  page_css: ""
+  page_xpath: ""
+  page_attr: href
+  fields:
+  - name: title
+    css: .rank_d_b_name > a
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: url
+    css: .rank_d_b_name > a
+    xpath: ""
+    attr: href
+    next_stage: ""
+    remark: ""
+  - name: abstract
+    css: body
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+  - name: votes
+    css: .rank_d_b_ticket
+    xpath: ""
+    attr: ""
+    next_stage: ""
+    remark: ""
+settings:
+  ROBOTSTXT_OBEY: "false"
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML,
+    like Gecko) Chrome/78.0.3904.108 Safari/537.36