From bfce15d54ee1b44c498e0369b0bd507187fc6063 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 26 Nov 2019 12:44:24 +0800
Subject: [PATCH 01/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5Spiderfile=E9=99=90?=
 =?UTF-8?q?=E5=88=B6=E6=9D=A1=E4=BB=B6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/services/config_spider.go | 39 ++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go
index 4e8005a1..91b474f7 100644
--- a/backend/services/config_spider.go
+++ b/backend/services/config_spider.go
@@ -37,12 +37,17 @@ func ValidateSpiderfile(configData entity.ConfigSpiderData) error {
 
 	// 校验是否存在 start_url
 	if configData.StartUrl == "" {
-		return errors.New("spiderfile start_url is empty")
+		return errors.New("spiderfile invalid: start_url is empty")
+	}
+
+	// 校验是否存在 start_stage
+	if configData.StartStage == "" {
+		return errors.New("spiderfile invalid: start_stage is empty")
 	}
 
 	// 校验是否存在 stages
 	if len(configData.Stages) == 0 {
-		return errors.New("spiderfile stages is empty")
+		return errors.New("spiderfile invalid: stages is empty")
 	}
 
 	// 校验stages
@@ -50,56 +55,64 @@ func ValidateSpiderfile(configData entity.ConfigSpiderData) error {
 	for stageName, stage := range configData.Stages {
 		// stage 名称不能为空
 		if stageName == "" {
-			return errors.New("spiderfile stage name is empty")
+			return errors.New("spiderfile invalid: stage name is empty")
 		}
 
 		// stage 名称不能为保留字符串
 		// NOTE: 如果有其他Engine，可以扩展，默认为Scrapy
 		if configData.Engine == "" || configData.Engine == constants.EngineScrapy {
 			if strings.Contains(constants.ScrapyProtectedStageNames, stageName) {
-				return errors.New(fmt.Sprintf("spiderfile stage name '%s' is protected", stageName))
+				return errors.New(fmt.Sprintf("spiderfile invalid: stage name '%s' is protected", stageName))
 			}
-		} else if configData.Engine == constants.EngineColly {
-			return errors.New(fmt.Sprintf("engine '%s' is not implemented", stageName))
+		} else {
+			return errors.New(fmt.Sprintf("spiderfile invalid: engine '%s' is not implemented", configData.Engine))
 		}
 
 		// stage 名称不能重复
 		if dict[stageName] == 1 {
-			return errors.New("spiderfile stage name should be unique")
+			return errors.New(fmt.Sprintf("spiderfile invalid: stage name '%s' is duplicated", stageName))
 		}
 		dict[stageName] = 1
 
 		// stage 字段不能为空
 		if len(stage.Fields) == 0 {
-			return errors.New(fmt.Sprintf("spiderfile stage '%s' has no fields", stageName))
+			return errors.New(fmt.Sprintf("spiderfile invalid: stage '%s' has no fields", stageName))
 		}
 
-		// stage 的下一个 stage 只能有一个
+		// 是否包含 next_stage
 		hasNextStage := false
+
+		// 遍历字段列表
 		for _, field := range stage.Fields {
+			// stage 的 next stage 只能有一个
 			if field.NextStage != "" {
 				if hasNextStage {
-					return errors.New("spiderfile stage fields should have only 1 next_stage")
+					return errors.New(fmt.Sprintf("spiderfile invalid: stage '%s' has more than 1 next_stage", stageName))
 				}
 				hasNextStage = true
 			}
+
+			// 字段里 CSS 和 XPath 只能包含一个
+			if field.Css != "" && field.Xpath != "" {
+				return errors.New(fmt.Sprintf("spiderfile invalid: field '%s' in stage '%s' has both CSS and XPath set which is prohibited", field.Name, stageName))
+			}
 		}
 
 		// 如果 stage 的 is_list 为 true 但 list_css 为空，报错
 		if stage.IsList && stage.ListCss == "" {
-			return errors.New("spiderfile stage with is_list = true should have list_css being set")
+			return errors.New("spiderfile invalid: stage with is_list = true should have list_css being set")
 		}
 	}
 
 	// 校验字段唯一性
 	if !IsUniqueConfigSpiderFields(fields) {
-		return errors.New("spiderfile fields not unique")
+		return errors.New("spiderfile invalid: fields not unique")
 	}
 
 	// 字段名称不能为保留字符串
 	for _, field := range fields {
 		if strings.Contains(constants.ScrapyProtectedFieldNames, field.Name) {
-			return errors.New(fmt.Sprintf("spiderfile field name '%s' is protected", field.Name))
+			return errors.New(fmt.Sprintf("spiderfile invalid: field name '%s' is protected", field.Name))
 		}
 	}
 

From b676a84e497c184c1d82b84e4220fed38305e436 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 26 Nov 2019 13:02:49 +0800
Subject: [PATCH 02/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5xpath?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/entity/config_spider.go       | 12 +++---
 backend/model/config_spider/scrapy.go | 62 ++++++++++++++++++---------
 backend/services/config_spider.go     | 14 +++++-
 3 files changed, 61 insertions(+), 27 deletions(-)

diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go
index 5e0fe1e1..d46e092e 100644
--- a/backend/entity/config_spider.go
+++ b/backend/entity/config_spider.go
@@ -9,11 +9,13 @@ type Field struct {
 }
 
 type Stage struct {
-	IsList   bool    `yaml:"is_list" json:"is_list"`
-	ListCss  string  `yaml:"list_css" json:"list_css"`
-	PageCss  string  `yaml:"page_css" json:"page_css"`
-	PageAttr string  `yaml:"page_attr" json:"page_attr"`
-	Fields   []Field `yaml:"fields" json:"fields"`
+	IsList    bool    `yaml:"is_list" json:"is_list"`
+	ListCss   string  `yaml:"list_css" json:"list_css"`
+	ListXpath string  `yaml:"list_xpath" json:"list_xpath"`
+	PageCss   string  `yaml:"page_css" json:"page_css"`
+	PageXpath string  `yaml:"page_xpath" json:"page_xpath"`
+	PageAttr  string  `yaml:"page_attr" json:"page_attr"`
+	Fields    []Field `yaml:"fields" json:"fields"`
 }
 
 type ConfigSpiderData struct {
diff --git a/backend/model/config_spider/scrapy.go b/backend/model/config_spider/scrapy.go
index 7503b9bf..9da6fb89 100644
--- a/backend/model/config_spider/scrapy.go
+++ b/backend/model/config_spider/scrapy.go
@@ -131,12 +131,7 @@ func (g ScrapyGenerator) GetNonListParserString(stageName string, stage entity.S
 
 	// 遍历字段列表
 	for _, f := range stage.Fields {
-		line := ""
-		if f.Attr == "" {
-			line += fmt.Sprintf(`item['%s'] = response.css('%s::text').extract_first()`, f.Name, f.Css)
-		} else {
-			line += fmt.Sprintf(`item['%s'] = response.css('%s::attr("%s")').extract_first()`, f.Name, f.Css, f.Attr)
-		}
+		line := fmt.Sprintf(`item['%s'] = response.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f))
 		line = g.PadCode(line, 2)
 		str += line
 	}
@@ -170,12 +165,7 @@ func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stag
 
 	// 遍历字段列表
 	for _, f := range stage.Fields {
-		line := ""
-		if f.Attr == "" {
-			line += fmt.Sprintf(`item['%s'] = elem.css('%s::text').extract_first()`, f.Name, f.Css)
-		} else {
-			line += fmt.Sprintf(`item['%s'] = elem.css('%s::attr("%s")').extract_first()`, f.Name, f.Css, f.Attr)
-		}
+		line := fmt.Sprintf(`item['%s'] = elem.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f))
 		line = g.PadCode(line, 3)
 		str += line
 	}
@@ -195,14 +185,8 @@ func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stag
 	}
 
 	// 分页
-	if stage.PageCss != "" {
-		// 分页元素属性，默认为 href
-		pageAttr := "href"
-		if stage.PageAttr != "" {
-			pageAttr = stage.PageAttr
-		}
-
-		str += g.PadCode(fmt.Sprintf(`next_url = response.css('%s::attr("%s")').extract_first()`, stage.PageCss, pageAttr), 2)
+	if stage.PageCss != "" || stage.PageXpath != "" {
+		str += g.PadCode(fmt.Sprintf(`next_url = response.%s.extract_first()`, g.GetExtractStringFromStage(stage)), 2)
 		str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': item})`, stageName), 2)
 	}
 
@@ -226,3 +210,41 @@ func (g ScrapyGenerator) GetNextStageField(stage entity.Stage) (entity.Field, er
 	}
 	return entity.Field{}, errors.New("cannot find next stage field")
 }
+
+func (g ScrapyGenerator) GetExtractStringFromField(f entity.Field) string {
+	if f.Css != "" {
+		// 如果为CSS
+		if f.Attr == "" {
+			// 文本
+			return fmt.Sprintf(`css(%s::text())`, f.Css)
+		} else {
+			// 属性
+			return fmt.Sprintf(`css(%s::attr("%s"))`, f.Css, f.Attr)
+		}
+	} else {
+		// 如果为XPath
+		if f.Attr == "" {
+			// 文本
+			return fmt.Sprintf(`xpath(%s/text())`, f.Xpath)
+		} else {
+			// 属性
+			return fmt.Sprintf(`xpath(%s/@%s)`, f.Xpath, f.Attr)
+		}
+	}
+}
+
+func (g ScrapyGenerator) GetExtractStringFromStage(stage entity.Stage) string {
+	// 分页元素属性，默认为 href
+	pageAttr := "href"
+	if stage.PageAttr != "" {
+		pageAttr = stage.PageAttr
+	}
+
+	if stage.PageCss != "" {
+		// 如果为CSS
+		return fmt.Sprintf(`css(%s::attr("%s"))`, stage.PageCss, pageAttr)
+	} else {
+		// 如果为XPath
+		return fmt.Sprintf(`xpath(%s/@%s)`, stage.PageXpath, pageAttr)
+	}
+}
diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go
index 91b474f7..35bb2790 100644
--- a/backend/services/config_spider.go
+++ b/backend/services/config_spider.go
@@ -92,12 +92,22 @@ func ValidateSpiderfile(configData entity.ConfigSpiderData) error {
 				hasNextStage = true
 			}
 
-			// 字段里 CSS 和 XPath 只能包含一个
+			// 字段里 css 和 xpath 只能包含一个
 			if field.Css != "" && field.Xpath != "" {
-				return errors.New(fmt.Sprintf("spiderfile invalid: field '%s' in stage '%s' has both CSS and XPath set which is prohibited", field.Name, stageName))
+				return errors.New(fmt.Sprintf("spiderfile invalid: field '%s' in stage '%s' has both css and xpath set which is prohibited", field.Name, stageName))
 			}
 		}
 
+		// stage 里 page_css 和 page_xpath 只能包含一个
+		if stage.PageCss != "" && stage.PageXpath != "" {
+			return errors.New(fmt.Sprintf("spiderfile invalid: stage '%s' has both page_css and page_xpath set which is prohibited", stageName))
+		}
+
+		// stage 里 list_css 和 list_xpath 只能包含一个
+		if stage.ListCss != "" && stage.ListXpath != "" {
+			return errors.New(fmt.Sprintf("spiderfile invalid: stage '%s' has both list_css and list_xpath set which is prohibited", stageName))
+		}
+
 		// 如果 stage 的 is_list 为 true 但 list_css 为空，报错
 		if stage.IsList && stage.ListCss == "" {
 			return errors.New("spiderfile invalid: stage with is_list = true should have list_css being set")

From b34ad260d95f709bfd8b7aa0bf34ac2896113536 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 26 Nov 2019 13:13:31 +0800
Subject: [PATCH 03/52] fixed eslint errors

---
 frontend/src/components/ScrollView/LogItem.vue | 1 -
 frontend/src/components/ScrollView/LogView.vue | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/frontend/src/components/ScrollView/LogItem.vue b/frontend/src/components/ScrollView/LogItem.vue
index c54d7c37..19e3e082 100644
--- a/frontend/src/components/ScrollView/LogItem.vue
+++ b/frontend/src/components/ScrollView/LogItem.vue
@@ -25,7 +25,6 @@ export default {
 }
 </script>
 
-
 <style scoped>
   .log-item {
     display: table;
diff --git a/frontend/src/components/ScrollView/LogView.vue b/frontend/src/components/ScrollView/LogView.vue
index fc19edb5..e07ca03b 100644
--- a/frontend/src/components/ScrollView/LogView.vue
+++ b/frontend/src/components/ScrollView/LogView.vue
@@ -15,7 +15,7 @@ import LogItem from './LogItem'
 import VirtualList from 'vue-virtual-scroll-list'
 import Convert from 'ansi-to-html'
 import hasAnsi from 'has-ansi'
-const convert = new Convert();
+const convert = new Convert()
 export default {
   name: 'LogView',
   components: {
@@ -53,7 +53,7 @@ export default {
         props: {
           index: logItem.index,
           data: isAnsi ? convert.toHtml(logItem.data) : logItem.data,
-          isAnsi,
+          isAnsi
         }
       }
     }

From 0d8ce3a29ff364d90bb99f39f2affe201ce17e8c Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 26 Nov 2019 13:38:54 +0800
Subject: [PATCH 04/52] =?UTF-8?q?=E5=87=86=E5=A4=87=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E5=90=8E=E7=AB=AF=E6=95=B0=E6=8D=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/main.go                 |  1 +
 backend/routes/config_spider.go | 47 +++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/backend/main.go b/backend/main.go
index 565c7892..8b91e950 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -140,6 +140,7 @@ func main() {
 			authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats)   // 爬虫统计数据
 			authGroup.GET("/spider/types", routes.GetSpiderTypes)        // 爬虫类型
 			// 可配置爬虫
+			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig)      // 可配置爬虫配置
 			authGroup.PUT("/config_spiders", routes.PutConfigSpider)                // 添加可配置爬虫
 			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)          // 修改可配置爬虫
 			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 6f4a2893..6261e2e5 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -236,3 +236,50 @@ func UploadConfigSpider(c *gin.Context) {
 		Message: "success",
 	})
 }
+
+func GetConfigSpiderConfig(c *gin.Context) {
+	id := c.Param("id")
+
+	// 校验ID
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "invalid id")
+	}
+
+	// 获取爬虫
+	spider, err := model.GetSpider(bson.ObjectIdHex(id))
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 校验爬虫类别
+	if spider.Type != constants.Configurable {
+		HandleErrorF(http.StatusBadRequest, c, "not a configurable spider")
+		return
+	}
+
+	// Spiderfile 目录
+	sfPath := filepath.Join(spider.Src, "Spiderfile")
+
+	// 构造配置数据
+	configData := entity.ConfigSpiderData{}
+
+	// 读取YAML文件
+	yamlFile, err := ioutil.ReadFile(sfPath)
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 反序列化
+	if err := yaml.Unmarshal(yamlFile, &configData); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    configData,
+	})
+}

From 2ab47e51b325d11eb62d41bac3adbffc58039563 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 26 Nov 2019 13:39:07 +0800
Subject: [PATCH 05/52] =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC?=
 =?UTF-8?q?=E8=99=AB=E5=89=8D=E7=AB=AF=E5=87=86=E5=A4=87?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../src/components/InfoView/SpiderInfoView.vue  | 13 ++++++-------
 frontend/src/i18n/zh.js                         |  3 +++
 frontend/src/views/spider/SpiderList.vue        | 17 ++++++++++++++---
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue
index 381b253c..50f1ccf8 100644
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -21,11 +21,11 @@
         <el-form-item :label="$t('Source Folder')">
           <el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
         </el-form-item>
-        <el-form-item  :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
+        <el-form-item v-if="spiderForm.type === 'customized'" :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
           <el-input v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"
                     :disabled="isView"></el-input>
         </el-form-item>
-        <el-form-item :label="$t('Results Collection')">
+        <el-form-item :label="$t('Results Collection')" prop="col" required :inline-message="true">
           <el-input v-model="spiderForm.col" :placeholder="$t('Results Collection')"
                     :disabled="isView"></el-input>
         </el-form-item>
@@ -39,11 +39,10 @@
           </el-autocomplete>
         </el-form-item>
         <el-form-item :label="$t('Spider Type')">
-          <!--<el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="true" clearable>-->
-            <!--<el-option value="configurable" :label="$t('Configurable')"></el-option>-->
-            <!--<el-option value="customized" :label="$t('Customized')"></el-option>-->
-          <!--</el-select>-->
-          <el-input v-model="spiderForm.type" placeholder="爬虫类型" clearable/>
+          <el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="true" clearable>
+            <el-option value="configurable" :label="$t('Configurable')"></el-option>
+            <el-option value="customized" :label="$t('Customized')"></el-option>
+          </el-select>
         </el-form-item>
         <el-form-item :label="$t('Remark')">
           <el-input v-model="spiderForm.remark"/>
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index c56959c9..43ef33ef 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -125,6 +125,8 @@ export default {
   'Customized Spider': '自定义爬虫',
   'Configurable': '可配置',
   'Customized': '自定义',
+  'configurable': '可配置',
+  'customized': '自定义',
   'Text': '文本',
   'Attribute': '属性',
   'Field Name': '字段名称',
@@ -148,6 +150,7 @@ export default {
   'List Page Fields': '列表页字段',
   'Detail Page Fields': '详情页字段',
   'Detail Page URL': '详情页URL',
+  'All': '全部',
 
   // 爬虫列表
   'Name': '名称',
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index eb1e548f..37ca04de 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -93,7 +93,8 @@
           </el-upload>
         </el-form-item>
       </el-form>
-      <el-alert type="error" :title="$t('Please zip your spider files from the root directory')" :closable="false"></el-alert>
+      <el-alert type="error" :title="$t('Please zip your spider files from the root directory')"
+                :closable="false"></el-alert>
     </el-dialog>
     <!--./customized spider dialog-->
 
@@ -143,6 +144,14 @@
       </div>
       <!--./filter-->
 
+      <!--tabs-->
+      <el-tabs v-model="activeTab" @tab-click="onClickTab">
+        <el-tab-pane :label="$t('All')" name="all"></el-tab-pane>
+        <el-tab-pane :label="$t('Configurable')" name="configurable"></el-tab-pane>
+        <el-tab-pane :label="$t('Customized')" name="customized"></el-tab-pane>
+      </el-tabs>
+      <!--./tabs-->
+
       <!--table list-->
       <el-table :data="spiderList"
                 class="table"
@@ -157,7 +166,7 @@
                            align="left"
                            :width="col.width">
             <template slot-scope="scope">
-              {{scope.row.type === 'customized' ? '自定义' : scope.row.type}}
+              {{$t(scope.row.type)}}
             </template>
           </el-table-column>
           <el-table-column v-else-if="col.name === 'last_5_errors'"
@@ -249,6 +258,7 @@ import dayjs from 'dayjs'
 import CrawlConfirmDialog from '../../components/Common/CrawlConfirmDialog'
 import StatusTag from '../../components/Status/StatusTag'
 import request from '../../api/request'
+
 export default {
   name: 'SpiderList',
   components: {
@@ -287,7 +297,8 @@ export default {
       spiderFormRules: {
         name: [{ required: true, message: 'Required Field', trigger: 'change' }]
       },
-      fileList: []
+      fileList: [],
+      activeTab: 'all'
     }
   },
   computed: {

From 95a1b72ec4e6a7c9237b74bf41e6a63d16460ffd Mon Sep 17 00:00:00 2001
From: zhangweiii <zhangwei.ink@gmail.com>
Date: Wed, 27 Nov 2019 09:45:16 +0800
Subject: [PATCH 06/52] fix Cannot read property length of null when first init

---
 backend/model/spider.go                      | 4 ++++
 frontend/src/views/schedule/ScheduleList.vue | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/backend/model/spider.go b/backend/model/spider.go
index 5c2c92e8..f2978125 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -116,6 +116,10 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, int, erro
 		return spiders, 0, err
 	}
 
+    if spiders == nil {
+        spiders = []Spider{}
+    }
+
 	// 遍历爬虫列表
 	for i, spider := range spiders {
 		// 获取最后一次任务
diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue
index b170c9ed..3a032b23 100644
--- a/frontend/src/views/schedule/ScheduleList.vue
+++ b/frontend/src/views/schedule/ScheduleList.vue
@@ -274,7 +274,7 @@ export default {
     // 爬虫列表
     request.get('/spiders', {})
       .then(response => {
-        this.spiderList = response.data.data.list
+        this.spiderList = response.data.data.list || []
       })
   }
 }

From 939b21b33edb2add1d0a4188d7dc80fc38c8ff62 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 10:05:46 +0800
Subject: [PATCH 07/52] added todo

---
 frontend/vue.config.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frontend/vue.config.js b/frontend/vue.config.js
index c867140e..f5de4ac5 100644
--- a/frontend/vue.config.js
+++ b/frontend/vue.config.js
@@ -1,3 +1,4 @@
 module.exports = {
   publicPath: process.env.BASE_URL || '/'
+  // TODO: need to configure output static files with hash
 }

From 1674dce3bd805f645547ddb1da12d5a83e3c9da6 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 10:40:37 +0800
Subject: [PATCH 08/52] =?UTF-8?q?=E5=87=86=E5=A4=87=E5=89=8D=E7=AB=AF?=
 =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC=E8=99=AB=E6=95=B0=E6=8D=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/main.go                 |  8 ++---
 backend/model/spider.go         | 56 ++++++++++++++++++++++++++++-----
 backend/routes/config_spider.go | 27 +---------------
 backend/routes/spider.go        |  2 +-
 4 files changed, 55 insertions(+), 38 deletions(-)

diff --git a/backend/main.go b/backend/main.go
index 8b91e950..436c537e 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -140,10 +140,10 @@ func main() {
 			authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats)   // 爬虫统计数据
 			authGroup.GET("/spider/types", routes.GetSpiderTypes)        // 爬虫类型
 			// 可配置爬虫
-			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig)      // 可配置爬虫配置
-			authGroup.PUT("/config_spiders", routes.PutConfigSpider)                // 添加可配置爬虫
-			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)          // 修改可配置爬虫
-			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
+			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 可配置爬虫配置
+			authGroup.PUT("/config_spiders", routes.PutConfigSpider)                  // 添加可配置爬虫
+			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)            // 修改可配置爬虫
+			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider)   // 上传可配置爬虫
 			// 任务
 			authGroup.GET("/tasks", routes.GetTaskList)                                 // 任务列表
 			authGroup.GET("/tasks/:id", routes.GetTask)                                 // 任务详情
diff --git a/backend/model/spider.go b/backend/model/spider.go
index 53c5ab1f..559234fc 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -1,11 +1,16 @@
 package model
 
 import (
+	"crawlab/constants"
 	"crawlab/database"
 	"crawlab/entity"
+	"errors"
 	"github.com/apex/log"
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"path/filepath"
 	"runtime/debug"
 	"time"
 )
@@ -25,14 +30,15 @@ type Spider struct {
 	Site        string        `json:"site" bson:"site"`                 // 爬虫网站
 	Envs        []Env         `json:"envs" bson:"envs"`                 // 环境变量
 	Remark      string        `json:"remark" bson:"remark"`             // 备注
+	Src         string        `json:"src" bson:"src"`                   // 源码位置
 
 	// 自定义爬虫
-	Src string `json:"src" bson:"src"` // 源码位置
 	Cmd string `json:"cmd" bson:"cmd"` // 执行命令
 
 	// 前端展示
-	LastRunTs  time.Time `json:"last_run_ts"` // 最后一次执行时间
-	LastStatus string    `json:"last_status"` // 最后执行状态
+	LastRunTs  time.Time               `json:"last_run_ts"` // 最后一次执行时间
+	LastStatus string                  `json:"last_status"` // 最后执行状态
+	Config     entity.ConfigSpiderData `json:"config"`      // 可配置爬虫配置
 
 	// 时间
 	CreateTs time.Time `json:"create_ts" bson:"create_ts"`
@@ -161,15 +167,25 @@ func GetSpider(id bson.ObjectId) (Spider, error) {
 	s, c := database.GetCol("spiders")
 	defer s.Close()
 
-	var result Spider
-	if err := c.FindId(id).One(&result); err != nil {
+	// 获取爬虫
+	var spider Spider
+	if err := c.FindId(id).One(&spider); err != nil {
 		if err != mgo.ErrNotFound {
 			log.Errorf("get spider error: %s, id: %id", err.Error(), id.Hex())
 			debug.PrintStack()
 		}
-		return result, err
+		return spider, err
 	}
-	return result, nil
+
+	// 如果为可配置爬虫，获取爬虫配置
+	if spider.Type == constants.Configurable {
+		config, err := GetConfigSpiderData(spider)
+		if err != nil {
+			return spider, err
+		}
+		spider.Config = config
+	}
+	return spider, nil
 }
 
 // 更新爬虫
@@ -269,3 +285,29 @@ func GetSpiderTypes() ([]*entity.SpiderType, error) {
 
 	return types, nil
 }
+
+func GetConfigSpiderData(spider Spider) (entity.ConfigSpiderData, error) {
+	// 构造配置数据
+	configData := entity.ConfigSpiderData{}
+
+	// 校验爬虫类别
+	if spider.Type != constants.Configurable {
+		return configData, errors.New("not a configurable spider")
+	}
+
+	// Spiderfile 目录
+	sfPath := filepath.Join(spider.Src, "Spiderfile")
+
+	// 读取YAML文件
+	yamlFile, err := ioutil.ReadFile(sfPath)
+	if err != nil {
+		return configData, err
+	}
+
+	// 反序列化
+	if err := yaml.Unmarshal(yamlFile, &configData); err != nil {
+		return configData, err
+	}
+
+	return configData, nil
+}
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 6261e2e5..085f5541 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -252,34 +252,9 @@ func GetConfigSpiderConfig(c *gin.Context) {
 		return
 	}
 
-	// 校验爬虫类别
-	if spider.Type != constants.Configurable {
-		HandleErrorF(http.StatusBadRequest, c, "not a configurable spider")
-		return
-	}
-
-	// Spiderfile 目录
-	sfPath := filepath.Join(spider.Src, "Spiderfile")
-
-	// 构造配置数据
-	configData := entity.ConfigSpiderData{}
-
-	// 读取YAML文件
-	yamlFile, err := ioutil.ReadFile(sfPath)
-	if err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
-		return
-	}
-
-	// 反序列化
-	if err := yaml.Unmarshal(yamlFile, &configData); err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
-		return
-	}
-
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
-		Data:    configData,
+		Data:    spider.Config,
 	})
 }
diff --git a/backend/routes/spider.go b/backend/routes/spider.go
index d351f1bb..588811e3 100644
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -34,7 +34,7 @@ func GetSpiderList(c *gin.Context) {
 		"name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}},
 	}
 
-	if t != "" {
+	if t != "" && t != "all" {
 		filter["type"] = t
 	}
 

From 0edda7b5585c28aab84bace26e2d98fa5fbed69d Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 12:18:51 +0800
Subject: [PATCH 09/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=89=8D=E7=AB=AF?=
 =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=BC=96=E8=BE=91=E5=99=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/entity/config_spider.go               |   1 +
 frontend/package.json                         |   2 +-
 frontend/src/components/Config/ConfigList.vue | 281 +++++++++++++-----
 .../components/TableView/FieldsTableView.vue  | 224 +++++++++++---
 frontend/src/views/spider/SpiderList.vue      |  38 +--
 frontend/yarn.lock                            |   7 +-
 6 files changed, 406 insertions(+), 147 deletions(-)

diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go
index d46e092e..095a4328 100644
--- a/backend/entity/config_spider.go
+++ b/backend/entity/config_spider.go
@@ -6,6 +6,7 @@ type Field struct {
 	Xpath     string `yaml:"xpath" json:"xpath"`
 	Attr      string `yaml:"attr" json:"attr"`
 	NextStage string `yaml:"next_stage" json:"next_stage"`
+	Remark    string `yaml:"remark" json:"remark"`
 }
 
 type Stage struct {
diff --git a/frontend/package.json b/frontend/package.json
index 5f19fd7b..724b5e36 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -23,7 +23,7 @@
     "cross-env": "^5.2.0",
     "dayjs": "^1.8.6",
     "echarts": "^4.1.0",
-    "element-ui": "2.4.6",
+    "element-ui": "2.13.0",
     "font-awesome": "^4.7.0",
     "js-cookie": "2.2.0",
     "normalize.css": "7.0.0",
diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 8cab5294..4ba92f3f 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -42,97 +42,159 @@
 
     <!--config detail-->
     <el-row>
-      <el-form label-width="150px" ref="form" :model="spiderForm">
+      <el-form label-width="150px" ref="form" :model="spiderForm.config">
         <el-col :span="11" :offset="1">
-          <el-form-item :label="$t('Crawl Type')">
-            <el-button-group>
-              <el-button v-for="type in crawlTypeList"
-                         :key="type.value"
-                         :type="type.value === spiderForm.crawl_type ? 'primary' : ''"
-                         @click="onSelectCrawlType(type.value)">
-                {{$t(type.label)}}
-              </el-button>
-            </el-button-group>
-          </el-form-item>
-          <el-form-item :label="$t('Start URL')" prop="start_url" required>
-            <el-input v-model="spiderForm.start_url" :placeholder="$t('Start URL')"></el-input>
-          </el-form-item>
-          <el-form-item :label="$t('Obey robots.txt')">
-            <el-switch v-model="spiderForm.obey_robots_txt" :placeholder="$t('Obey robots.txt')"></el-switch>
-          </el-form-item>
+          <!--          <el-form-item :label="$t('Crawl Type')">-->
+          <!--            <el-button-group>-->
+          <!--              <el-button v-for="type in crawlTypeList"-->
+          <!--                         :key="type.value"-->
+          <!--                         :type="type.value === spiderForm.crawl_type ? 'primary' : ''"-->
+          <!--                         @click="onSelectCrawlType(type.value)">-->
+          <!--                {{$t(type.label)}}-->
+          <!--              </el-button>-->
+          <!--            </el-button-group>-->
+          <!--          </el-form-item>-->
+          <!--          <el-form-item :label="$t('Start URL')" prop="start_url" required>-->
+          <!--            <el-input v-model="spiderForm.config.start_url" :placeholder="$t('Start URL')"></el-input>-->
+          <!--          </el-form-item>-->
+          <!--          <el-form-item :label="$t('Obey robots.txt')">-->
+          <!--            <el-switch v-model="spiderForm.obey_robots_txt" :placeholder="$t('Obey robots.txt')"></el-switch>-->
+          <!--          </el-form-item>-->
           <!--<el-form-item :label="$t('URL Pattern')">-->
           <!--<el-input v-model="spiderForm.url_pattern" :placeholder="$t('URL Pattern')"></el-input>-->
           <!--</el-form-item>-->
         </el-col>
         <el-col :span="11" :offset="1">
-          <el-form-item :label="$t('Item Selector')"
-                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
-            <el-select style="width: 35%;margin-right: 10px;"
-                       v-model="spiderForm.item_selector_type"
-                       :placeholder="$t('Item Selector Type')">
-              <el-option value="xpath" :label="$t('XPath')"></el-option>
-              <el-option value="css" :label="$t('CSS')"></el-option>
-            </el-select>
-            <el-input style="width: calc(65% - 10px);"
-                      v-model="spiderForm.item_selector"
-                      :placeholder="$t('Item Selector')">
-            </el-input>
-          </el-form-item>
-          <el-form-item :label="$t('Pagination Selector')"
-                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
-            <el-select style="width: 35%;margin-right: 10px;"
-                       v-model="spiderForm.pagination_selector_type"
-                       :placeholder="$t('Pagination Selector Type')">
-              <el-option value="xpath" :label="$t('XPath')"></el-option>
-              <el-option value="css" :label="$t('CSS')"></el-option>
-            </el-select>
-            <el-input style="width: calc(65% - 10px);"
-                      v-model="spiderForm.pagination_selector"
-                      :placeholder="$t('Pagination Selector')">
-            </el-input>
-          </el-form-item>
-          <el-form-item :label="$t('Item Threshold')"
-                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
-            <el-input-number v-model="spiderForm.item_threshold"/>
-          </el-form-item>
+          <!--          <el-form-item :label="$t('Item Selector')"-->
+          <!--                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">-->
+          <!--            <el-select style="width: 35%;margin-right: 10px;"-->
+          <!--                       v-model="spiderForm.item_selector_type"-->
+          <!--                       :placeholder="$t('Item Selector Type')">-->
+          <!--              <el-option value="xpath" :label="$t('XPath')"></el-option>-->
+          <!--              <el-option value="css" :label="$t('CSS')"></el-option>-->
+          <!--            </el-select>-->
+          <!--            <el-input style="width: calc(65% - 10px);"-->
+          <!--                      v-model="spiderForm.item_selector"-->
+          <!--                      :placeholder="$t('Item Selector')">-->
+          <!--            </el-input>-->
+          <!--          </el-form-item>-->
+          <!--          <el-form-item :label="$t('Pagination Selector')"-->
+          <!--                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">-->
+          <!--            <el-select style="width: 35%;margin-right: 10px;"-->
+          <!--                       v-model="spiderForm.page"-->
+          <!--                       :placeholder="$t('Pagination Selector Type')">-->
+          <!--              <el-option value="xpath" :label="$t('XPath')"></el-option>-->
+          <!--              <el-option value="css" :label="$t('CSS')"></el-option>-->
+          <!--            </el-select>-->
+          <!--            <el-input style="width: calc(65% - 10px);"-->
+          <!--                      v-model="spiderForm.pagination_selector"-->
+          <!--                      :placeholder="$t('Pagination Selector')">-->
+          <!--            </el-input>-->
+          <!--          </el-form-item>-->
+          <!--          <el-form-item :label="$t('Item Threshold')"-->
+          <!--                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">-->
+          <!--            <el-input-number v-model="spiderForm.item_threshold"/>-->
+          <!--          </el-form-item>-->
         </el-col>
       </el-form>
     </el-row>
     <!--./config detail-->
 
-    <!--button group-->
-    <el-row class="button-group-container">
-      <div class="button-group">
-        <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
-        <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">{{$t('Extract Fields')}}
-        </el-button>
-        <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
-        <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
-      </div>
-    </el-row>
-    <!--./button group-->
+    <el-row>
+      <el-col :span="12">
+        <el-row>
+          <span class="selector-type-item" @click="onClickSelectorType('css')">
+              <el-tag
+                :class="isCss ? 'active' : 'inactive'"
+                type="success"
+              >
+                CSS
+              </el-tag>
+            </span>
+          <span class="selector-type-item" @click="onClickSelectorType('xpath')">
+              <el-tag
+                :class="isXpath ? 'active' : 'inactive'"
+                type="primary"
+              >
+                XPath
+              </el-tag>
+            </span>
+        </el-row>
+      </el-col>
 
-    <!--list field list-->
-    <el-row v-if="['list','list-detail'].includes(spiderForm.crawl_type)"
-            class="list-fields-container">
-      <fields-table-view
-        type="list"
-        title="List Page Fields"
-        :fields="spiderForm.fields"
-      />
+      <el-col :span="12">
+        <!--button group-->
+        <el-row class="button-group-container">
+          <div class="button-group">
+            <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
+            <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">{{$t('Extract Fields')}}
+            </el-button>
+            <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
+            <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
+          </div>
+        </el-row>
+        <!--./button group-->
+      </el-col>
     </el-row>
+
+    <el-collapse
+      v-model="activeNames"
+    >
+      <el-collapse-item
+        v-for="(stage, stageName) in spiderForm.config.stages"
+        :key="stageName"
+      >
+        <template slot="title">
+          {{stageName}}
+        </template>
+        <fields-table-view
+          type="list"
+          title="List Page Fields"
+          :fields="stage.fields"
+          :stage-names="Object.keys(spiderForm.config.stages)"
+        />
+        <!--        <el-table-->
+        <!--          :data="stage.fields"-->
+        <!--        >-->
+        <!--          <el-table-column-->
+        <!--            v-for="fCol in fieldColumns"-->
+        <!--            :key="fCol.name"-->
+        <!--            :label="$t(fCol.label)"-->
+        <!--          >-->
+        <!--            <template slot-scope="scope">-->
+        <!--              <template v-if="fCol.name === 'selector_type'">-->
+        <!--                <el-tag :class="scope.row.css ? 'active' : 'inactive'" type="success">CSS</el-tag>-->
+        <!--                <el-tag :class="scope.row.xpath ? 'active' : 'inactive'" type="primary">XPath</el-tag>-->
+        <!--              </template>-->
+        <!--              <template v-else>-->
+        <!--                {{scope.row[fCol.name]}}-->
+        <!--              </template>-->
+        <!--            </template>-->
+        <!--          </el-table-column>-->
+        <!--        </el-table>-->
+      </el-collapse-item>
+    </el-collapse>
+    <!--list field list-->
+    <!--    <el-row v-if="['list','list-detail'].includes(spiderForm.crawl_type)"-->
+    <!--            class="list-fields-container">-->
+    <!--    <fields-table-view-->
+    <!--      type="list"-->
+    <!--      title="List Page Fields"-->
+    <!--      :fields="spiderForm.fields"-->
+    <!--    />-->
+    <!--        </el-row>-->
     <!--./list field list-->
 
     <!--detail field list-->
-    <el-row v-if="['detail','list-detail'].includes(spiderForm.crawl_type)"
-            class="detail-fields-container"
-            style="margin-top: 10px;">
-      <fields-table-view
-        type="detail"
-        title="Detail Page Fields"
-        :fields="spiderForm.detail_fields"
-      />
-    </el-row>
+    <!--    <el-row v-if="['detail','list-detail'].includes(spiderForm.crawl_type)"-->
+    <!--            class="detail-fields-container"-->
+    <!--            style="margin-top: 10px;">-->
+    <!--      <fields-table-view-->
+    <!--        type="detail"-->
+    <!--        title="Detail Page Fields"-->
+    <!--        :fields="spiderForm.detail_fields"-->
+    <!--      />-->
+    <!--    </el-row>-->
     <!--./detail field list-->
   </div>
 </template>
@@ -146,9 +208,13 @@ import CrawlConfirmDialog from '../Common/CrawlConfirmDialog'
 
 export default {
   name: 'ConfigList',
-  components: { CrawlConfirmDialog, FieldsTableView },
+  components: {
+    CrawlConfirmDialog,
+    FieldsTableView
+  },
   data () {
     return {
+      activeNames: [],
       crawlTypeList: [
         { value: 'list', label: 'List Only' },
         { value: 'detail', label: 'Detail Only' },
@@ -159,7 +225,15 @@ export default {
       saveLoading: false,
       dialogVisible: false,
       crawlConfirmDialogVisible: false,
-      columnsDict: {}
+      columnsDict: {},
+      fieldColumns: [
+        { name: 'name', label: 'Name' },
+        { name: 'selector_type', label: 'Selector Type' },
+        { name: 'selector', label: 'Selector' },
+        { name: 'is_attr', label: 'Is Attribute' },
+        { name: 'attr', label: 'Attribute' },
+        { name: 'next_stage', label: 'Next Stage' }
+      ]
     }
   },
   computed: {
@@ -177,6 +251,24 @@ export default {
       } else {
         return []
       }
+    },
+    isCss () {
+      let i = 0
+      Object.values(this.spiderForm.config.stages).forEach(stage => {
+        stage.fields.forEach(field => {
+          if (!field.css) i++
+        })
+      })
+      return i === 0
+    },
+    isXpath () {
+      let i = 0
+      Object.values(this.spiderForm.config.stages).forEach(stage => {
+        stage.fields.forEach(field => {
+          if (!field.xpath) i++
+        })
+      })
+      return i === 0
     }
   },
   methods: {
@@ -294,7 +386,20 @@ export default {
       value = value.trim()
       if (value.length > 20) return value.substr(0, 20) + '...'
       return value
-    }
+    },
+    onClickSelectorType (selectorType) {
+      Object.values(this.spiderForm.config.stages).forEach(stage => {
+        stage.fields.forEach(field => {
+          if (selectorType === 'css') {
+            if (field.xpath) field.xpath = ''
+            if (!field.css) field.css = 'body'
+          } else {
+            if (field.css) field.css = ''
+            if (!field.xpath) field.xpath = '//body'
+          }
+        })
+      })
+    },
   },
   created () {
     // fields for list page
@@ -327,6 +432,9 @@ export default {
     if (!this.spiderForm.pagination_selector_type) this.$set(this.spiderForm, 'pagination_selector_type', 'css')
     if (this.spiderForm.obey_robots_txt == null) this.$set(this.spiderForm, 'obey_robots_txt', true)
     if (this.spiderForm.item_threshold == null) this.$set(this.spiderForm, 'item_threshold', 10)
+  },
+  mounted () {
+    this.activeNames = Object.keys(this.spiderForm.config.stages)
   }
 }
 </script>
@@ -335,7 +443,7 @@ export default {
 
   .button-group-container {
     margin-top: 10px;
-    border-bottom: 1px dashed #dcdfe6;
+    /*border-bottom: 1px dashed #dcdfe6;*/
     padding-bottom: 20px;
   }
 
@@ -345,7 +453,7 @@ export default {
 
   .list-fields-container {
     margin-top: 20px;
-    border-bottom: 1px dashed #dcdfe6;
+    /*border-bottom: 1px dashed #dcdfe6;*/
     padding-bottom: 20px;
   }
 
@@ -369,4 +477,17 @@ export default {
   .el-table.table-header >>> .el-input .el-input__inner {
     border-radius: 0;
   }
+
+  .selector-type-item {
+    margin: 0 5px;
+    cursor: pointer;
+  }
+
+  .selector-type-item > .el-tag.active {
+    font-weight: bolder;
+  }
+
+  .selector-type-item > .el-tag.inactive {
+    opacity: 0.5;
+  }
 </style>
diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue
index 836a8f49..7b1ef9f5 100644
--- a/frontend/src/components/TableView/FieldsTableView.vue
+++ b/frontend/src/components/TableView/FieldsTableView.vue
@@ -1,73 +1,104 @@
 <template>
   <div class="fields-table-view">
-    <el-row class="button-group-container">
-      <label class="title">{{$t(this.title)}}</label>
-      <div class="button-group">
-        <el-button type="primary" size="small" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>
-      </div>
-    </el-row>
+    <!--    <el-row class="button-group-container">-->
+    <!--      <label class="title">{{$t(this.title)}}</label>-->
+    <!--      <div class="button-group">-->
+    <!--        <el-button type="primary" size="small" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>-->
+    <!--      </div>-->
+    <!--    </el-row>-->
     <el-row>
       <el-table :data="fields"
                 class="table edit"
                 :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
-                border>
-        <el-table-column v-if="type === 'list' && spiderForm.crawl_type === 'list-detail'"
-                         :label="$t('Detail Page URL')"
-                         align="center">
+      >
+        <el-table-column class-name="action" width="80px" align="right">
           <template slot-scope="scope">
-            <el-checkbox v-model="scope.row.is_detail"
-                         @change="onCheck(scope.row)">
-            </el-checkbox>
+            <i class="action-item el-icon-copy-document" @click="onCopyField(scope.row)"></i>
+            <i class="action-item el-icon-remove-outline" @click="onRemoveField(scope.row)"></i>
+            <i class="action-item el-icon-circle-plus-outline" @click="onAddField(scope.row)"></i>
           </template>
         </el-table-column>
-        <el-table-column :label="$t('Field Name')" width="200px">
+        <el-table-column :label="$t('Field Name')" width="150px">
           <template slot-scope="scope">
             <el-input v-model="scope.row.name" :placeholder="$t('Field Name')"
                       @change="onNameChange(scope.row)"></el-input>
           </template>
         </el-table-column>
-        <el-table-column :label="$t('Query Type')" width="200px">
+        <el-table-column :label="$t('Selector Type')" width="150px" align="center" class-name="selector-type">
           <template slot-scope="scope">
-            <el-select v-model="scope.row.type" :placeholder="$t('Query Type')">
-              <el-option value="css" :label="$t('CSS Selector')"></el-option>
-              <el-option value="xpath" :label="$t('XPath')"></el-option>
-            </el-select>
+            <span class="button-selector-item" @click="onClickSelectorType(scope.row, 'css')">
+              <el-tag
+                :class="scope.row.css ? 'active' : 'inactive'"
+                type="success"
+              >
+                CSS
+              </el-tag>
+            </span>
+            <span class="button-selector-item" @click="onClickSelectorType(scope.row, 'xpath')">
+              <el-tag
+                :class="scope.row.xpath ? 'active' : 'inactive'"
+                type="primary"
+              >
+                XPath
+              </el-tag>
+            </span>
           </template>
         </el-table-column>
-        <el-table-column :label="$t('Query')" width="250px">
+        <el-table-column :label="$t('Selector')" width="200px">
           <template slot-scope="scope">
-            <el-input v-model="scope.row.query" :placeholder="$t('Query')"></el-input>
-          </template>
-        </el-table-column>
-        <el-table-column :label="$t('Extract Type')" width="120px">
-          <template slot-scope="scope">
-            <el-select v-model="scope.row.extract_type" :placeholder="$t('Extract Type')">
-              <el-option value="text" :label="$t('Text')"></el-option>
-              <el-option value="attribute" :label="$t('Attribute')"></el-option>
-            </el-select>
-          </template>
-        </el-table-column>
-        <el-table-column :label="$t('Attribute')" width="250px">
-          <template slot-scope="scope">
-            <template v-if="scope.row.extract_type === 'attribute'">
-              <el-input v-model="scope.row.attribute"
-                        :placeholder="$t('Attribute')">
-              </el-input>
+            <template v-if="scope.row.css">
+              <el-input v-model="scope.row.css" :placeholder="$t('CSS')"></el-input>
             </template>
             <template v-else>
+              <el-input v-model="scope.row.xpath" :placeholder="$t('XPath')"></el-input>
             </template>
           </template>
         </el-table-column>
-        <el-table-column :label="$t('Action')" fixed="right" min-width="100px">
+        <el-table-column :label="$t('Is Attribute')" width="150px" align="center">
           <template slot-scope="scope">
-            <div class="action-button-group">
-              <el-button size="mini"
-                         style="margin-left:10px"
-                         icon="el-icon-delete"
-                         type="danger"
-                         @click="deleteField(scope.$index)">
-              </el-button>
-            </div>
+            <span class="button-selector-item" @click="onClickIsAttribute(scope.row, false)">
+              <el-tag
+                :class="!scope.row.attr ? 'active' : 'inactive'"
+                type="success"
+              >
+                {{$t('Text')}}
+              </el-tag>
+            </span>
+            <span class="button-selector-item" @click="onClickIsAttribute(scope.row, true)">
+              <el-tag
+                :class="scope.row.attr ? 'active' : 'inactive'"
+                type="primary"
+              >
+                {{$t('Attribute')}}
+              </el-tag>
+            </span>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Attribute')" width="200px">
+          <template slot-scope="scope">
+            <template v-if="scope.row.attr">
+              <el-input v-model="scope.row.attr" :placeholder="$t('Attribute')"/>
+            </template>
+            <template v-else>
+              <span style="margin-left: 15px">
+                N/A
+              </span>
+            </template>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Next Stage')" width="250px">
+          <template slot-scope="scope">
+            <el-select
+              v-model="scope.row.next_stage"
+            >
+              <el-option :label="$t('No Next Stage')" value=""/>
+              <el-option v-for="n in stageNames" :key="n" :label="n" :value="n"/>
+            </el-select>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Remark')" width="auto" min-width="120px">
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.remark" :placeholder="$t('Remark')"/>
           </template>
         </el-table-column>
       </el-table>
@@ -91,6 +122,12 @@ export default {
       type: String,
       default: ''
     },
+    stageNames: {
+      type: Array,
+      default () {
+        return []
+      }
+    },
     fields: {
       type: Array,
       default () {
@@ -128,6 +165,57 @@ export default {
         }
       })
       this.$st.sendEv('爬虫详情-配置', '设置详情页URL')
+    },
+    onClickSelectorType (row, selectorType) {
+      if (selectorType === 'css') {
+        if (row.xpath) row.xpath = ''
+        if (!row.css) row.css = 'body'
+      } else {
+        if (row.css) row.css = ''
+        if (!row.xpath) row.xpath = '//body'
+      }
+    },
+    onClickIsAttribute (row, isAttribute) {
+      if (!isAttribute) {
+        // 文本
+        if (row.attr) row.attr = ''
+      } else {
+        // 属性
+        if (!row.attr) row.attr = 'href'
+      }
+    },
+    onCopyField (row) {
+      for (let i = 0; i < this.fields.length; i++) {
+        if (row.name === this.fields[i].name) {
+          this.fields.splice(i, 0, JSON.parse(JSON.stringify(row)))
+          break
+        }
+      }
+    },
+    onRemoveField (row) {
+      for (let i = 0; i < this.fields.length; i++) {
+        if (row.name === this.fields[i].name) {
+          this.fields.splice(i, 1)
+          break
+        }
+      }
+      if (this.fields.length === 0) {
+        this.fields.push({
+          css: 'body',
+          next_stage: ''
+        })
+      }
+    },
+    onAddField (row) {
+      for (let i = 0; i < this.fields.length; i++) {
+        if (row.name === this.fields[i].name) {
+          this.fields.splice(i + 1, 0, {
+            css: 'body',
+            next_stage: ''
+          })
+          break
+        }
+      }
     }
   }
 }
@@ -158,6 +246,50 @@ export default {
     line-height: 36px;
   }
 
+  .el-table.edit >>> .button-selector-item {
+    cursor: pointer;
+    margin: 0 5px;
+  }
+
+  .el-table.edit >>> .el-tag.inactive {
+    opacity: 0.5;
+  }
+
+  .el-table.edit >>> .action {
+    background: none !important;
+    border: none;
+  }
+
+  .el-table.edit >>> tr {
+    border: none;
+  }
+
+  .el-table.edit >>> tr th {
+    border-right: 1px solid rgb(220, 223, 230);
+  }
+
+  .el-table.edit >>> tr td:nth-child(2) {
+    border-left: 1px solid rgb(220, 223, 230);
+  }
+
+  .el-table.edit >>> tr td {
+    border-right: 1px solid rgb(220, 223, 230);
+  }
+
+  .el-table.edit::before {
+    background: none;
+  }
+
+  .el-table.edit >>> .action-item {
+    font-size: 14px;
+    margin-right: 5px;
+    cursor: pointer;
+  }
+
+  .el-table.edit >>> .action-item:last-child {
+    margin-right: 10px;
+  }
+
   .button-group-container {
     /*display: inline-block;*/
     /*width: 100%;*/
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index 37ca04de..22563796 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -111,17 +111,24 @@
       <div class="filter">
         <div class="left">
           <el-form :inline="true">
-            <el-form-item>
-              <el-select clearable @change="onSpiderTypeChange" placeholder="爬虫类型" size="small" v-model="filter.type">
-                <el-option v-for="item in types" :value="item.type" :key="item.type"
-                           :label="item.type === 'customized'? '自定义':item.type "/>
-              </el-select>
-            </el-form-item>
+            <!--            <el-form-item>-->
+            <!--              <el-select clearable @change="onSpiderTypeChange" placeholder="爬虫类型" size="small" v-model="filter.type">-->
+            <!--                <el-option v-for="item in types" :value="item.type" :key="item.type"-->
+            <!--                           :label="item.type === 'customized'? '自定义':item.type "/>-->
+            <!--              </el-select>-->
+            <!--            </el-form-item>-->
             <el-form-item>
               <el-input clearable @keyup.enter.native="onSearch" size="small" placeholder="名称" v-model="filter.keyword">
                 <i slot="suffix" class="el-input__icon el-icon-search"></i>
               </el-input>
             </el-form-item>
+            <el-form-item>
+              <el-button size="small" type="success"
+                         class="btn refresh"
+                         @click="onRefresh">
+                {{$t('Search')}}
+              </el-button>
+            </el-form-item>
           </el-form>
         </div>
         <div class="right">
@@ -134,18 +141,13 @@
                      @click="onAdd">
             {{$t('Add Spider')}}
           </el-button>
-          <el-button size="small" type="success"
-                     icon="el-icon-refresh"
-                     class="btn refresh"
-                     @click="onRefresh">
-            {{$t('Refresh')}}
-          </el-button>
+
         </div>
       </div>
       <!--./filter-->
 
       <!--tabs-->
-      <el-tabs v-model="activeTab" @tab-click="onClickTab">
+      <el-tabs v-model="filter.type" @tab-click="onClickTab">
         <el-tab-pane :label="$t('All')" name="all"></el-tab-pane>
         <el-tab-pane :label="$t('Configurable')" name="configurable"></el-tab-pane>
         <el-tab-pane :label="$t('Customized')" name="customized"></el-tab-pane>
@@ -282,10 +284,9 @@ export default {
       activeSpiderId: undefined,
       filter: {
         keyword: '',
-        type: ''
+        type: 'all'
       },
       types: [],
-      // tableData,
       columns: [
         { name: 'display_name', label: 'Name', width: '160', align: 'left' },
         { name: 'type', label: 'Spider Type', width: '120' },
@@ -297,8 +298,7 @@ export default {
       spiderFormRules: {
         name: [{ required: true, message: 'Required Field', trigger: 'change' }]
       },
-      fileList: [],
-      activeTab: 'all'
+      fileList: []
     }
   },
   computed: {
@@ -499,6 +499,10 @@ export default {
         this.onView(row)
       }
     },
+    onClickTab (tab) {
+      this.filter.type = tab.name
+      this.getList()
+    },
     getList () {
       let params = {
         pageNum: this.pagination.pageNum,
diff --git a/frontend/yarn.lock b/frontend/yarn.lock
index 8697e8e2..ef361b2b 100644
--- a/frontend/yarn.lock
+++ b/frontend/yarn.lock
@@ -2954,9 +2954,10 @@ electron-to-chromium@^1.3.103:
   version "1.3.113"
   resolved "http://registry.npm.taobao.org/electron-to-chromium/download/electron-to-chromium-1.3.113.tgz#b1ccf619df7295aea17bc6951dc689632629e4a9"
 
-element-ui@2.4.6:
-  version "2.4.6"
-  resolved "https://registry.yarnpkg.com/element-ui/-/element-ui-2.4.6.tgz#524d3d4cac0b68745dda87311ef0d8fe541b5fc4"
+element-ui@2.13.0:
+  version "2.13.0"
+  resolved "https://registry.npm.taobao.org/element-ui/download/element-ui-2.13.0.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Felement-ui%2Fdownload%2Felement-ui-2.13.0.tgz#f6bb04e5b0a76ea5f62466044b774407ba4ebd2d"
+  integrity sha1-9rsE5bCnbqX2JGYES3dEB7pOvS0=
   dependencies:
     async-validator "~1.8.1"
     babel-helper-vue-jsx-merge-props "^2.0.0"

From 6449a8c5be73ed9cc8f4df2a9116df06f117ad46 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 12:32:25 +0800
Subject: [PATCH 10/52] =?UTF-8?q?=E5=87=86=E5=A4=87=E5=89=8D=E7=AB=AF?=
 =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC=E8=99=AB=E6=95=B0=E6=8D=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/entity/config_spider.go | 1 +
 backend/model/spider.go         | 5 +++++
 backend/routes/config_spider.go | 5 +++++
 3 files changed, 11 insertions(+)

diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go
index 095a4328..7afb7109 100644
--- a/backend/entity/config_spider.go
+++ b/backend/entity/config_spider.go
@@ -10,6 +10,7 @@ type Field struct {
 }
 
 type Stage struct {
+	Name      string  `yaml:"name" json:"name"`
 	IsList    bool    `yaml:"is_list" json:"is_list"`
 	ListCss   string  `yaml:"list_css" json:"list_css"`
 	ListXpath string  `yaml:"list_xpath" json:"list_xpath"`
diff --git a/backend/model/spider.go b/backend/model/spider.go
index 559234fc..f2072b8d 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -309,5 +309,10 @@ func GetConfigSpiderData(spider Spider) (entity.ConfigSpiderData, error) {
 		return configData, err
 	}
 
+	// 赋值 stage_name
+	for stageName, stage := range configData.Stages {
+		stage.Name = stageName
+	}
+
 	return configData, nil
 }
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 085f5541..5fa25f2c 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -151,6 +151,11 @@ func UploadConfigSpider(c *gin.Context) {
 		return
 	}
 
+	// 赋值 stage_name
+	for stageName, stage := range configData.Stages {
+		stage.Name = stageName
+	}
+
 	// 删除已有的爬虫文件
 	for _, fInfo := range utils.ListDir(spiderDir) {
 		// 不删除Spiderfile

From b3c34c6f2a69c5e20832c44e7a177146075f666d Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 12:41:32 +0800
Subject: [PATCH 11/52] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=89=8D=E7=AB=AF?=
 =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC=E8=99=AB=E7=BC=96=E8=BE=91?=
 =?UTF-8?q?=E5=99=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 44 ++++++++++++++++++-
 1 file changed, 43 insertions(+), 1 deletion(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 4ba92f3f..a41fcd96 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -145,7 +145,21 @@
         :key="stageName"
       >
         <template slot="title">
-          {{stageName}}
+          <ul class="stage-list">
+            <li class="stage-item">
+              <label>{{$t('Stage Name')}}: </label>
+              <el-input v-model="stage.name" :placeholder="$t('Stage Name')" @focus="onStageNameFocus($event)"/>
+            </li>
+          </ul>
+<!--          <el-table :data="[{}]" :show-header="false" height="120px">-->
+<!--            <el-table-column width="100px">{{$t('Stage Name')}}:</el-table-column>-->
+<!--            <el-table-column width="180px">{{stageName}}</el-table-column>-->
+<!--          </el-table>-->
+<!--          <el-form label-width="120px" inline>-->
+<!--            <el-form-item :label="$t('Stage Name')">-->
+<!--              {{$t(stageName)}}-->
+<!--            </el-form-item>-->
+<!--          </el-form>-->
         </template>
         <fields-table-view
           type="list"
@@ -400,6 +414,11 @@ export default {
         })
       })
     },
+    onStageNameFocus (ev) {
+      console.log(ev)
+      // ev.stopPropagation()
+      ev.preventDefault()
+    }
   },
   created () {
     // fields for list page
@@ -490,4 +509,27 @@ export default {
   .selector-type-item > .el-tag.inactive {
     opacity: 0.5;
   }
+
+  .stage-list {
+    display: flex;
+    list-style: none;
+  }
+
+  .stage-list .stage-item {
+    flex-basis: 360px;
+    display: flex;
+  }
+
+  .stage-list .stage-item label {
+    width: 180px;
+  }
+
+  .stage-list .stage-item .el-input {
+    height: 32px;
+  }
+
+  .stage-list .stage-item .el-input .el-input__inner {
+    height: 32px;
+    inline-size: 32px;
+  }
 </style>

From e88b13b510769e7cd43bd2132be33446c8627a90 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 12:42:08 +0800
Subject: [PATCH 12/52] =?UTF-8?q?=E5=87=86=E5=A4=87=E5=89=8D=E7=AB=AF?=
 =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC=E8=99=AB=E6=95=B0=E6=8D=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/spider.go         | 7 ++++---
 backend/routes/config_spider.go | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/backend/model/spider.go b/backend/model/spider.go
index 43ff2de6..1fe0617f 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -114,9 +114,9 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, int, erro
 		return spiders, 0, err
 	}
 
-    if spiders == nil {
-        spiders = []Spider{}
-    }
+	if spiders == nil {
+		spiders = []Spider{}
+	}
 
 	// 遍历爬虫列表
 	for i, spider := range spiders {
@@ -316,6 +316,7 @@ func GetConfigSpiderData(spider Spider) (entity.ConfigSpiderData, error) {
 	// 赋值 stage_name
 	for stageName, stage := range configData.Stages {
 		stage.Name = stageName
+		configData.Stages[stageName] = stage
 	}
 
 	return configData, nil
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 5fa25f2c..cdd40607 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -154,6 +154,7 @@ func UploadConfigSpider(c *gin.Context) {
 	// 赋值 stage_name
 	for stageName, stage := range configData.Stages {
 		stage.Name = stageName
+		configData.Stages[stageName] = stage
 	}
 
 	// 删除已有的爬虫文件

From 88b7da10cb844e55fdc5239218a46df6f47aee38 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 15:26:50 +0800
Subject: [PATCH 13/52] =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC?=
 =?UTF-8?q?=E8=99=AB=E5=89=8D=E7=AB=AF=E6=A0=B7=E5=BC=8F=E8=B0=83=E6=95=B4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 156 +++++++++++-------
 1 file changed, 98 insertions(+), 58 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index a41fcd96..cc405950 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -138,28 +138,40 @@
     </el-row>
 
     <el-collapse
-      v-model="activeNames"
+      :value="activeNames"
     >
       <el-collapse-item
         v-for="(stage, stageName) in spiderForm.config.stages"
         :key="stageName"
+        :name="stageName"
       >
         <template slot="title">
           <ul class="stage-list">
-            <li class="stage-item">
+            <li class="stage-item" style="flex-basis: 80px; justify-content: flex-end"
+                @click="$event.stopPropagation()">
+              <i class="action-item el-icon-copy-document" @click="onCopyStage(stage)"></i>
+              <i class="action-item el-icon-remove-outline" @click="onRemoveStage(stage)"></i>
+              <i class="action-item el-icon-circle-plus-outline" @click="onAddStage(stage)"></i>
+            </li>
+            <li class="stage-item" @click="$event.stopPropagation()">
               <label>{{$t('Stage Name')}}: </label>
-              <el-input v-model="stage.name" :placeholder="$t('Stage Name')" @focus="onStageNameFocus($event)"/>
+              <div v-if="!stage.isEdit" @click="onEditStage(stage)" class="text-wrapper">
+                <span class="text">
+                  {{stage.name}}
+                </span>
+                <i class="el-icon-edit-outline"></i>
+              </div>
+              <el-input
+                v-else
+                :ref="`stage-name-${stage.name}`"
+                class="edit-text"
+                v-model="stage.name"
+                :placeholder="$t('Stage Name')"
+                @focus="onStageNameFocus($event)"
+                @blur="stage.isEdit = false"
+              />
             </li>
           </ul>
-<!--          <el-table :data="[{}]" :show-header="false" height="120px">-->
-<!--            <el-table-column width="100px">{{$t('Stage Name')}}:</el-table-column>-->
-<!--            <el-table-column width="180px">{{stageName}}</el-table-column>-->
-<!--          </el-table>-->
-<!--          <el-form label-width="120px" inline>-->
-<!--            <el-form-item :label="$t('Stage Name')">-->
-<!--              {{$t(stageName)}}-->
-<!--            </el-form-item>-->
-<!--          </el-form>-->
         </template>
         <fields-table-view
           type="list"
@@ -167,49 +179,8 @@
           :fields="stage.fields"
           :stage-names="Object.keys(spiderForm.config.stages)"
         />
-        <!--        <el-table-->
-        <!--          :data="stage.fields"-->
-        <!--        >-->
-        <!--          <el-table-column-->
-        <!--            v-for="fCol in fieldColumns"-->
-        <!--            :key="fCol.name"-->
-        <!--            :label="$t(fCol.label)"-->
-        <!--          >-->
-        <!--            <template slot-scope="scope">-->
-        <!--              <template v-if="fCol.name === 'selector_type'">-->
-        <!--                <el-tag :class="scope.row.css ? 'active' : 'inactive'" type="success">CSS</el-tag>-->
-        <!--                <el-tag :class="scope.row.xpath ? 'active' : 'inactive'" type="primary">XPath</el-tag>-->
-        <!--              </template>-->
-        <!--              <template v-else>-->
-        <!--                {{scope.row[fCol.name]}}-->
-        <!--              </template>-->
-        <!--            </template>-->
-        <!--          </el-table-column>-->
-        <!--        </el-table>-->
       </el-collapse-item>
     </el-collapse>
-    <!--list field list-->
-    <!--    <el-row v-if="['list','list-detail'].includes(spiderForm.crawl_type)"-->
-    <!--            class="list-fields-container">-->
-    <!--    <fields-table-view-->
-    <!--      type="list"-->
-    <!--      title="List Page Fields"-->
-    <!--      :fields="spiderForm.fields"-->
-    <!--    />-->
-    <!--        </el-row>-->
-    <!--./list field list-->
-
-    <!--detail field list-->
-    <!--    <el-row v-if="['detail','list-detail'].includes(spiderForm.crawl_type)"-->
-    <!--            class="detail-fields-container"-->
-    <!--            style="margin-top: 10px;">-->
-    <!--      <fields-table-view-->
-    <!--        type="detail"-->
-    <!--        title="Detail Page Fields"-->
-    <!--        :fields="spiderForm.detail_fields"-->
-    <!--      />-->
-    <!--    </el-row>-->
-    <!--./detail field list-->
   </div>
 </template>
 
@@ -228,7 +199,6 @@ export default {
   },
   data () {
     return {
-      activeNames: [],
       crawlTypeList: [
         { value: 'list', label: 'List Only' },
         { value: 'detail', label: 'Detail Only' },
@@ -283,6 +253,9 @@ export default {
         })
       })
       return i === 0
+    },
+    activeNames () {
+      return Object.values(this.spiderForm.config.stages).map(d => d.name)
     }
   },
   methods: {
@@ -416,8 +389,27 @@ export default {
     },
     onStageNameFocus (ev) {
       console.log(ev)
-      // ev.stopPropagation()
-      ev.preventDefault()
+      ev.stopPropagation()
+      // ev.preventDefault()
+    },
+    onEditStage (stage) {
+      this.$set(stage, 'isEdit', !stage.isEdit)
+      setTimeout(() => {
+        this.$refs[`stage-name-${stage.name}`][0].focus()
+      }, 0)
+    },
+    onCopyStage (stage) {
+      const ts = Math.floor(new Date().getTime()).toString()
+      const newStage = JSON.parse(JSON.stringify(stage))
+      newStage.name = `stage_${ts}`
+      this.$set(this.spiderForm.config.stages, newStage.name, newStage)
+    },
+    onRemoveStage (stage) {
+      const stages = JSON.parse(JSON.stringify(this.spiderForm.config.stages))
+      delete stages[stage.name]
+      this.$set(this.spiderForm.config, 'stages', stages)
+    },
+    onAddStage (stage) {
     }
   },
   created () {
@@ -511,20 +503,29 @@ export default {
   }
 
   .stage-list {
+    width: calc(80px + 320px);
     display: flex;
+    flex-wrap: wrap;
     list-style: none;
+    margin: 0;
+    padding: 0;
   }
 
   .stage-list .stage-item {
-    flex-basis: 360px;
+    flex-basis: 320px;
+    width: 320px;
     display: flex;
+    align-items: center;
   }
 
   .stage-list .stage-item label {
-    width: 180px;
+    flex-basis: 90px;
+    margin-right: 10px;
+    justify-self: flex-end;
   }
 
   .stage-list .stage-item .el-input {
+    flex-basis: calc(100% - 90px);
     height: 32px;
   }
 
@@ -532,4 +533,43 @@ export default {
     height: 32px;
     inline-size: 32px;
   }
+
+  .stage-list .stage-item .action-item {
+    cursor: pointer;
+    width: 13px;
+    margin-right: 5px;
+  }
+
+  .stage-list .stage-item .action-item:last-child {
+    margin-right: 10px;
+  }
+
+  .stage-list .stage-item .text-wrapper {
+    display: flex;
+    align-items: center;
+    max-width: calc(100% - 90px - 10px);
+  }
+
+  .stage-list .stage-item .text-wrapper .text {
+    text-overflow: ellipsis;
+    overflow: hidden;
+  }
+
+  .stage-list .stage-item .text-wrapper .text:hover {
+    text-decoration: underline;
+  }
+
+  .stage-list .stage-item .text-wrapper i {
+    margin-left: 5px;
+  }
+
+  .stage-list .stage-item >>> .edit-text {
+    height: 32px;
+    line-height: 32px;
+  }
+
+  .stage-list .stage-item >>> .edit-text .el-input__inner {
+    height: 32px;
+    line-height: 32px;
+  }
 </style>

From 27508699d22dc6525c091f315c89c11e533c45aa Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 19:59:38 +0800
Subject: [PATCH 14/52] =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC?=
 =?UTF-8?q?=E8=99=AB=E5=89=8D=E7=AB=AF=E6=A0=B7=E5=BC=8F=E8=B0=83=E6=95=B4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 218 +++++++++++-------
 .../components/TableView/FieldsTableView.vue  |  41 +++-
 2 files changed, 167 insertions(+), 92 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index cc405950..e54469a0 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -43,75 +43,63 @@
     <!--config detail-->
     <el-row>
       <el-form label-width="150px" ref="form" :model="spiderForm.config">
-        <el-col :span="11" :offset="1">
-          <!--          <el-form-item :label="$t('Crawl Type')">-->
-          <!--            <el-button-group>-->
-          <!--              <el-button v-for="type in crawlTypeList"-->
-          <!--                         :key="type.value"-->
-          <!--                         :type="type.value === spiderForm.crawl_type ? 'primary' : ''"-->
-          <!--                         @click="onSelectCrawlType(type.value)">-->
-          <!--                {{$t(type.label)}}-->
-          <!--              </el-button>-->
-          <!--            </el-button-group>-->
-          <!--          </el-form-item>-->
-          <!--          <el-form-item :label="$t('Start URL')" prop="start_url" required>-->
-          <!--            <el-input v-model="spiderForm.config.start_url" :placeholder="$t('Start URL')"></el-input>-->
-          <!--          </el-form-item>-->
-          <!--          <el-form-item :label="$t('Obey robots.txt')">-->
-          <!--            <el-switch v-model="spiderForm.obey_robots_txt" :placeholder="$t('Obey robots.txt')"></el-switch>-->
-          <!--          </el-form-item>-->
-          <!--<el-form-item :label="$t('URL Pattern')">-->
-          <!--<el-input v-model="spiderForm.url_pattern" :placeholder="$t('URL Pattern')"></el-input>-->
-          <!--</el-form-item>-->
-        </el-col>
-        <el-col :span="11" :offset="1">
-          <!--          <el-form-item :label="$t('Item Selector')"-->
-          <!--                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">-->
-          <!--            <el-select style="width: 35%;margin-right: 10px;"-->
-          <!--                       v-model="spiderForm.item_selector_type"-->
-          <!--                       :placeholder="$t('Item Selector Type')">-->
-          <!--              <el-option value="xpath" :label="$t('XPath')"></el-option>-->
-          <!--              <el-option value="css" :label="$t('CSS')"></el-option>-->
-          <!--            </el-select>-->
-          <!--            <el-input style="width: calc(65% - 10px);"-->
-          <!--                      v-model="spiderForm.item_selector"-->
-          <!--                      :placeholder="$t('Item Selector')">-->
-          <!--            </el-input>-->
-          <!--          </el-form-item>-->
-          <!--          <el-form-item :label="$t('Pagination Selector')"-->
-          <!--                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">-->
-          <!--            <el-select style="width: 35%;margin-right: 10px;"-->
-          <!--                       v-model="spiderForm.page"-->
-          <!--                       :placeholder="$t('Pagination Selector Type')">-->
-          <!--              <el-option value="xpath" :label="$t('XPath')"></el-option>-->
-          <!--              <el-option value="css" :label="$t('CSS')"></el-option>-->
-          <!--            </el-select>-->
-          <!--            <el-input style="width: calc(65% - 10px);"-->
-          <!--                      v-model="spiderForm.pagination_selector"-->
-          <!--                      :placeholder="$t('Pagination Selector')">-->
-          <!--            </el-input>-->
-          <!--          </el-form-item>-->
-          <!--          <el-form-item :label="$t('Item Threshold')"-->
-          <!--                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">-->
-          <!--            <el-input-number v-model="spiderForm.item_threshold"/>-->
-          <!--          </el-form-item>-->
-        </el-col>
       </el-form>
     </el-row>
     <!--./config detail-->
 
     <el-row>
-      <el-col :span="12">
-        <el-row>
-          <span class="selector-type-item" @click="onClickSelectorType('css')">
-              <el-tag
-                :class="isCss ? 'active' : 'inactive'"
-                type="success"
-              >
-                CSS
-              </el-tag>
-            </span>
-          <span class="selector-type-item" @click="onClickSelectorType('xpath')">
+      <div class="top-wrapper">
+        <ul class="list">
+          <li class="item">
+            <label>{{$t('Start URL')}}: </label>
+            <el-input
+              v-model="spiderForm.config.start_url"
+              :placeholder="$t('Start URL')"
+              :class="startUrlClass"
+            />
+          </li>
+          <li class="item">
+            <label>{{$t('Start Stage')}}: </label>
+            <el-select
+              v-model="spiderForm.config.start_stage"
+              :placeholder="$t('Start Stage')"
+              :class="startStageClass"
+            >
+              <el-option
+                v-for="n in Object.keys(spiderForm.config.stages)"
+                :key="n"
+                :value="n"
+                :label="n"
+              />
+            </el-select>
+          </li>
+          <li class="item">
+            <label>{{$t('Engine')}}: </label>
+            <el-select
+              v-model="spiderForm.config.engine"
+              :placeholder="$t('Start Stage')"
+              :class="startStageClass"
+            >
+              <el-option
+                v-for="n in Object.keys(spiderForm.config.stages)"
+                :key="n"
+                :value="n"
+                :label="n"
+              />
+            </el-select>
+          </li>
+          <li class="item">
+            <label>{{$t('Selector Type')}}: </label>
+            <div class="selector-type">
+              <span class="selector-type-item" @click="onClickSelectorType('css')">
+                <el-tag
+                  :class="isCss ? 'active' : 'inactive'"
+                  type="success"
+                >
+                  CSS
+                </el-tag>
+              </span>
+              <span class="selector-type-item" @click="onClickSelectorType('xpath')">
               <el-tag
                 :class="isXpath ? 'active' : 'inactive'"
                 type="primary"
@@ -119,22 +107,20 @@
                 XPath
               </el-tag>
             </span>
-        </el-row>
-      </el-col>
+            </div>
+          </li>
+        </ul>
+      </div>
 
-      <el-col :span="12">
-        <!--button group-->
-        <el-row class="button-group-container">
-          <div class="button-group">
-            <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
-            <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">{{$t('Extract Fields')}}
-            </el-button>
-            <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
-            <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
-          </div>
-        </el-row>
-        <!--./button group-->
-      </el-col>
+      <div class="button-group-container">
+        <div class="button-group">
+          <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
+          <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">{{$t('Extract Fields')}}
+          </el-button>
+          <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
+          <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
+        </div>
+      </div>
     </el-row>
 
     <el-collapse
@@ -256,6 +242,23 @@ export default {
     },
     activeNames () {
       return Object.values(this.spiderForm.config.stages).map(d => d.name)
+    },
+    startUrlClass () {
+      if (!this.spiderForm.config.start_url) {
+        return 'invalid'
+      } else if (!this.spiderForm.config.start_url.match(/^https?:\/\/.+|^\/\/.+/i)) {
+        return 'invalid'
+      }
+
+      return ''
+    },
+    startStageClass () {
+      if (!this.spiderForm.config.start_stage) {
+        return 'invalid'
+      } else if (!this.activeNames.includes(this.spiderForm.config.start_stage)) {
+        return 'invalid'
+      }
+      return ''
     }
   },
   methods: {
@@ -408,8 +411,27 @@ export default {
       const stages = JSON.parse(JSON.stringify(this.spiderForm.config.stages))
       delete stages[stage.name]
       this.$set(this.spiderForm.config, 'stages', stages)
+      if (Object.keys(stages).length === 0) {
+        this.onAddStage()
+      }
     },
     onAddStage (stage) {
+      const stages = JSON.parse(JSON.stringify(this.spiderForm.config.stages))
+      const ts = Math.floor(new Date().getTime()).toString()
+      const newStageName = `stage_${ts}`
+      const newField = { name: `field_${ts}`, next_stage: '' }
+      if (this.isCss) {
+        newField['css'] = 'body'
+      } else if (this.isXpath) {
+        newField['xpath'] = '//body'
+      } else {
+        newField['css'] = 'body'
+      }
+      stages[newStageName] = {
+        name: newStageName,
+        fields: [newField]
+      }
+      this.$set(this.spiderForm.config, 'stages', stages)
     }
   },
   created () {
@@ -492,9 +514,6 @@ export default {
   .selector-type-item {
     margin: 0 5px;
     cursor: pointer;
-  }
-
-  .selector-type-item > .el-tag.active {
     font-weight: bolder;
   }
 
@@ -572,4 +591,39 @@ export default {
     height: 32px;
     line-height: 32px;
   }
+
+  .top-wrapper {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+  }
+
+  .top-wrapper .list {
+    list-style: none;
+    display: flex;
+    flex-wrap: wrap;
+    align-items: center;
+    padding: 0;
+  }
+
+  .top-wrapper .list .item {
+    margin-bottom: 10px;
+    display: flex;
+    align-items: center;
+  }
+
+  .top-wrapper .list .item label {
+    width: 100px;
+    text-align: right;
+    margin-right: 10px;
+    font-size: 12px;
+  }
+
+  .top-wrapper .list .item label + * {
+    width: 240px;
+  }
+
+  .invalid >>> .el-input__inner {
+    border: 1px solid red !important;
+  }
 </style>
diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue
index 7b1ef9f5..220deda8 100644
--- a/frontend/src/components/TableView/FieldsTableView.vue
+++ b/frontend/src/components/TableView/FieldsTableView.vue
@@ -10,6 +10,7 @@
       <el-table :data="fields"
                 class="table edit"
                 :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
+                :cell-style="getCellClassStyle"
       >
         <el-table-column class-name="action" width="80px" align="right">
           <template slot-scope="scope">
@@ -20,8 +21,10 @@
         </el-table-column>
         <el-table-column :label="$t('Field Name')" width="150px">
           <template slot-scope="scope">
-            <el-input v-model="scope.row.name" :placeholder="$t('Field Name')"
-                      @change="onNameChange(scope.row)"></el-input>
+            <el-input v-model="scope.row.name"
+                      :placeholder="$t('Field Name')"
+                      @change="onNameChange(scope.row)"
+            />
           </template>
         </el-table-column>
         <el-table-column :label="$t('Selector Type')" width="150px" align="center" class-name="selector-type">
@@ -47,10 +50,10 @@
         <el-table-column :label="$t('Selector')" width="200px">
           <template slot-scope="scope">
             <template v-if="scope.row.css">
-              <el-input v-model="scope.row.css" :placeholder="$t('CSS')"></el-input>
+              <el-input v-model="scope.row.css" :placeholder="$t('CSS / XPath')"></el-input>
             </template>
             <template v-else>
-              <el-input v-model="scope.row.xpath" :placeholder="$t('XPath')"></el-input>
+              <el-input v-model="scope.row.xpath" :placeholder="$t('CSS / XPath')"></el-input>
             </template>
           </template>
         </el-table-column>
@@ -168,20 +171,20 @@ export default {
     },
     onClickSelectorType (row, selectorType) {
       if (selectorType === 'css') {
-        if (row.xpath) row.xpath = ''
-        if (!row.css) row.css = 'body'
+        if (row.xpath) this.$set(row, 'xpath', '')
+        if (!row.css) this.$set(row, 'css', 'body')
       } else {
-        if (row.css) row.css = ''
-        if (!row.xpath) row.xpath = '//body'
+        if (row.css) this.$set(row, 'css', '')
+        if (!row.xpath) this.$set(row, 'xpath', '//body')
       }
     },
     onClickIsAttribute (row, isAttribute) {
       if (!isAttribute) {
         // 文本
-        if (row.attr) row.attr = ''
+        if (row.attr) this.$set(row, 'attr', '')
       } else {
         // 属性
-        if (!row.attr) row.attr = 'href'
+        if (!row.attr) this.$set(row, 'attr', 'href')
       }
     },
     onCopyField (row) {
@@ -210,12 +213,30 @@ export default {
       for (let i = 0; i < this.fields.length; i++) {
         if (row.name === this.fields[i].name) {
           this.fields.splice(i + 1, 0, {
+            name: `field_${Math.floor(new Date().getTime()).toString()}`,
             css: 'body',
             next_stage: ''
           })
           break
         }
       }
+    },
+    getCellClassStyle ({ row, columnIndex }) {
+      if (columnIndex === 1) {
+        // 字段名称
+        if (!row.name) {
+          return {
+            'border': '1px solid red'
+          }
+        }
+      } else if (columnIndex === 3) {
+        // 选择器
+        if (!row.css && !row.xpath) {
+          return {
+            'border': '1px solid red'
+          }
+        }
+      }
     }
   }
 }

From 4e8bd8a06b26d688e96b29e629f44900d6699c63 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 27 Nov 2019 20:33:25 +0800
Subject: [PATCH 15/52] =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC?=
 =?UTF-8?q?=E8=99=AB=E5=8A=A0=E5=85=A5=E5=8F=AF=E8=A7=86=E5=8C=96?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 327 ++++++++++++------
 1 file changed, 220 insertions(+), 107 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index e54469a0..1b03c335 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -40,57 +40,62 @@
     />
     <!--./crawl confirm dialog-->
 
-    <!--config detail-->
-    <el-row>
-      <el-form label-width="150px" ref="form" :model="spiderForm.config">
-      </el-form>
-    </el-row>
-    <!--./config detail-->
+    <!--tabs-->
+    <el-tabs :active-name="activeTab" @tab-click="onTabClick">
+      <!--Stages-->
+      <el-tab-pane name="stages" :label="$t('Stages')">
+        <!--config detail-->
+        <el-row>
+          <el-form label-width="150px" ref="form" :model="spiderForm.config">
+          </el-form>
+        </el-row>
+        <!--./config detail-->
 
-    <el-row>
-      <div class="top-wrapper">
-        <ul class="list">
-          <li class="item">
-            <label>{{$t('Start URL')}}: </label>
-            <el-input
-              v-model="spiderForm.config.start_url"
-              :placeholder="$t('Start URL')"
-              :class="startUrlClass"
-            />
-          </li>
-          <li class="item">
-            <label>{{$t('Start Stage')}}: </label>
-            <el-select
-              v-model="spiderForm.config.start_stage"
-              :placeholder="$t('Start Stage')"
-              :class="startStageClass"
-            >
-              <el-option
-                v-for="n in Object.keys(spiderForm.config.stages)"
-                :key="n"
-                :value="n"
-                :label="n"
-              />
-            </el-select>
-          </li>
-          <li class="item">
-            <label>{{$t('Engine')}}: </label>
-            <el-select
-              v-model="spiderForm.config.engine"
-              :placeholder="$t('Start Stage')"
-              :class="startStageClass"
-            >
-              <el-option
-                v-for="n in Object.keys(spiderForm.config.stages)"
-                :key="n"
-                :value="n"
-                :label="n"
-              />
-            </el-select>
-          </li>
-          <li class="item">
-            <label>{{$t('Selector Type')}}: </label>
-            <div class="selector-type">
+        <el-row>
+          <div class="top-wrapper">
+            <ul class="list">
+              <li class="item">
+                <label>{{$t('Start URL')}}: </label>
+                <el-input
+                  v-model="spiderForm.config.start_url"
+                  :placeholder="$t('Start URL')"
+                  :class="startUrlClass"
+                />
+              </li>
+              <li class="item">
+                <label>{{$t('Start Stage')}}: </label>
+                <el-select
+                  v-model="spiderForm.config.start_stage"
+                  :placeholder="$t('Start Stage')"
+                  :class="startStageClass"
+                >
+                  <el-option
+                    v-for="n in Object.keys(spiderForm.config.stages)"
+                    :key="n"
+                    :value="n"
+                    :label="n"
+                  />
+                </el-select>
+              </li>
+              <li class="item">
+                <label>{{$t('Engine')}}: </label>
+                <el-select
+                  v-model="spiderForm.config.engine"
+                  :placeholder="$t('Start Stage')"
+                  :class="startStageClass"
+                  disabled
+                >
+                  <el-option
+                    v-for="n in ['scrapy']"
+                    :key="n"
+                    :value="n"
+                    :label="n"
+                  />
+                </el-select>
+              </li>
+              <li class="item">
+                <label>{{$t('Selector Type')}}: </label>
+                <div class="selector-type">
               <span class="selector-type-item" @click="onClickSelectorType('css')">
                 <el-tag
                   :class="isCss ? 'active' : 'inactive'"
@@ -99,7 +104,7 @@
                   CSS
                 </el-tag>
               </span>
-              <span class="selector-type-item" @click="onClickSelectorType('xpath')">
+                  <span class="selector-type-item" @click="onClickSelectorType('xpath')">
               <el-tag
                 :class="isXpath ? 'active' : 'inactive'"
                 type="primary"
@@ -107,66 +112,77 @@
                 XPath
               </el-tag>
             </span>
+                </div>
+              </li>
+            </ul>
+          </div>
+
+          <div class="button-group-container">
+            <div class="button-group">
+              <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
+              <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">
+                {{$t('ExtractFields')}}
+              </el-button>
+              <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
+              <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
             </div>
-          </li>
-        </ul>
-      </div>
+          </div>
+        </el-row>
 
-      <div class="button-group-container">
-        <div class="button-group">
-          <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
-          <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">{{$t('Extract Fields')}}
-          </el-button>
-          <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
-          <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
-        </div>
-      </div>
-    </el-row>
-
-    <el-collapse
-      :value="activeNames"
-    >
-      <el-collapse-item
-        v-for="(stage, stageName) in spiderForm.config.stages"
-        :key="stageName"
-        :name="stageName"
-      >
-        <template slot="title">
-          <ul class="stage-list">
-            <li class="stage-item" style="flex-basis: 80px; justify-content: flex-end"
-                @click="$event.stopPropagation()">
-              <i class="action-item el-icon-copy-document" @click="onCopyStage(stage)"></i>
-              <i class="action-item el-icon-remove-outline" @click="onRemoveStage(stage)"></i>
-              <i class="action-item el-icon-circle-plus-outline" @click="onAddStage(stage)"></i>
-            </li>
-            <li class="stage-item" @click="$event.stopPropagation()">
-              <label>{{$t('Stage Name')}}: </label>
-              <div v-if="!stage.isEdit" @click="onEditStage(stage)" class="text-wrapper">
+        <el-collapse
+          :value="activeNames"
+        >
+          <el-collapse-item
+            v-for="(stage, stageName) in spiderForm.config.stages"
+            :key="stageName"
+            :name="stageName"
+          >
+            <template slot="title">
+              <ul class="stage-list">
+                <li class="stage-item" style="flex-basis: 80px; justify-content: flex-end"
+                    @click="$event.stopPropagation()">
+                  <i class="action-item el-icon-copy-document" @click="onCopyStage(stage)"></i>
+                  <i class="action-item el-icon-remove-outline" @click="onRemoveStage(stage)"></i>
+                  <i class="action-item el-icon-circle-plus-outline" @click="onAddStage(stage)"></i>
+                </li>
+                <li class="stage-item" @click="$event.stopPropagation()">
+                  <label>{{$t('Stage Name')}}: </label>
+                  <div v-if="!stage.isEdit" @click="onEditStage(stage)" class="text-wrapper">
                 <span class="text">
                   {{stage.name}}
                 </span>
-                <i class="el-icon-edit-outline"></i>
-              </div>
-              <el-input
-                v-else
-                :ref="`stage-name-${stage.name}`"
-                class="edit-text"
-                v-model="stage.name"
-                :placeholder="$t('Stage Name')"
-                @focus="onStageNameFocus($event)"
-                @blur="stage.isEdit = false"
-              />
-            </li>
-          </ul>
-        </template>
-        <fields-table-view
-          type="list"
-          title="List Page Fields"
-          :fields="stage.fields"
-          :stage-names="Object.keys(spiderForm.config.stages)"
-        />
-      </el-collapse-item>
-    </el-collapse>
+                    <i class="el-icon-edit-outline"></i>
+                  </div>
+                  <el-input
+                    v-else
+                    :ref="`stage-name-${stage.name}`"
+                    class="edit-text"
+                    v-model="stage.name"
+                    :placeholder="$t('Stage Name')"
+                    @focus="onStageNameFocus($event)"
+                    @blur="stage.isEdit = false"
+                  />
+                </li>
+              </ul>
+            </template>
+            <fields-table-view
+              type="list"
+              title="List Page Fields"
+              :fields="stage.fields"
+              :stage-names="Object.keys(spiderForm.config.stages)"
+            />
+          </el-collapse-item>
+        </el-collapse>
+      </el-tab-pane>
+      <!--./Stages-->
+
+      <!--Graph-->
+      <el-tab-pane name="process" :label="$t('Process')">
+        <div id="process-chart"></div>
+      </el-tab-pane>
+      <!--./Graph-->
+    </el-tabs>
+    <!--./tabs-->
   </div>
 </template>
 
@@ -174,6 +190,7 @@
 import {
   mapState
 } from 'vuex'
+import echarts from 'echarts'
 import FieldsTableView from '../TableView/FieldsTableView'
 import CrawlConfirmDialog from '../Common/CrawlConfirmDialog'
 
@@ -183,6 +200,13 @@ export default {
     CrawlConfirmDialog,
     FieldsTableView
   },
+  watch: {
+    activeTab () {
+      setTimeout(() => {
+        this.renderProcessChart()
+      }, 0)
+    }
+  },
   data () {
     return {
       crawlTypeList: [
@@ -203,7 +227,9 @@ export default {
         { name: 'is_attr', label: 'Is Attribute' },
         { name: 'attr', label: 'Attribute' },
         { name: 'next_stage', label: 'Next Stage' }
-      ]
+      ],
+      activeTab: 'stages',
+      processChart: undefined
     }
   },
   computed: {
@@ -259,6 +285,35 @@ export default {
         return 'invalid'
       }
       return ''
+    },
+    stageNodes () {
+      const elChart = document.querySelector('#process-chart')
+      const totalWidth = Number(getComputedStyle(elChart).width.replace('px', ''))
+      const stages = Object.values(this.spiderForm.config.stages)
+      return stages.map((stage, i) => {
+        return {
+          name: stage.name,
+          // x: i * totalWidth / stages.length,
+          // y: 0,
+          ...stage
+        }
+      })
+    },
+    stageEdges () {
+      const edges = []
+      const stages = Object.values(this.spiderForm.config.stages)
+      stages.forEach(stage => {
+        for (let i = 0; i < stage.fields.length; i++) {
+          const field = stage.fields[i]
+          if (field.next_stage) {
+            edges.push({
+              source: stage.name,
+              target: field.next_stage
+            })
+          }
+        }
+      })
+      return edges
     }
   },
   methods: {
@@ -432,6 +487,59 @@ export default {
         fields: [newField]
       }
       this.$set(this.spiderForm.config, 'stages', stages)
+    },
+    renderProcessChart () {
+      const option = {
+        title: {
+          text: this.$t('Stage Process')
+        },
+        series: [
+          {
+            type: 'graph',
+            layout: 'force',
+            symbolSize: 50,
+            roam: true,
+            label: {
+              normal: {
+                show: true
+              }
+            },
+            edgeSymbol: ['circle', 'arrow'],
+            edgeSymbolSize: [4, 10],
+            edgeLabel: {
+              normal: {
+                textStyle: {
+                  fontSize: 20
+                }
+              }
+            },
+            focusOneNodeAdjacency: true,
+            force: {
+              initLayout: 'force',
+              repulsion: 100,
+              gravity: 0.01,
+              edgeLength: 200
+            },
+            // draggable: true,
+            data: this.stageNodes,
+            links: this.stageEdges,
+            lineStyle: {
+              normal: {
+                opacity: 0.9,
+                width: 2,
+                curveness: 0
+              }
+            }
+          }
+        ]
+      }
+      const el = document.querySelector('#process-chart')
+      this.processChart = echarts.init(el)
+      this.processChart.setOption(option)
+      this.processChart.resize()
+    },
+    onTabClick (tab) {
+      this.activeTab = tab.name
     }
   },
   created () {
@@ -626,4 +734,9 @@ export default {
   .invalid >>> .el-input__inner {
     border: 1px solid red !important;
   }
+
+  #process-chart {
+    width: 100%;
+    height: 480px;
+  }
 </style>

From 429135eb4684278c1ae0cde0143a4d61d9d3edeb Mon Sep 17 00:00:00 2001
From: hantmac <hantmac@outlook.com>
Date: Wed, 27 Nov 2019 23:05:35 +0800
Subject: [PATCH 16/52] bug fix

---
 backend/services/spider_handler/spider.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go
index cce025dc..3ce42d52 100644
--- a/backend/services/spider_handler/spider.go
+++ b/backend/services/spider_handler/spider.go
@@ -119,6 +119,14 @@ func (s *SpiderSync) Download() {
 		return
 	}
 
+	//修改目标文件夹权限
+	// 解决scrapy.setting中开启LOG_ENABLED 和 LOG_FILE时不能创建log文件的问题
+	if err := os.Chmod(dstPath, 0777); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return
+	}
+
 	// 关闭临时文件
 	if err := tmpFile.Close(); err != nil {
 		log.Errorf(err.Error())

From 389da906ef1106ea8fddf7152e8e15ebfaa53364 Mon Sep 17 00:00:00 2001
From: hantmac <hantmac@outlook.com>
Date: Thu, 28 Nov 2019 10:26:14 +0800
Subject: [PATCH 17/52] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=BA=E9=80=92?=
 =?UTF-8?q?=E5=BD=92=E4=BF=AE=E6=94=B9=E6=96=87=E4=BB=B6=E5=A4=B9=E6=9D=83?=
 =?UTF-8?q?=E9=99=90?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/services/spider_handler/spider.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go
index 3ce42d52..e8ec6a19 100644
--- a/backend/services/spider_handler/spider.go
+++ b/backend/services/spider_handler/spider.go
@@ -10,6 +10,7 @@ import (
 	"github.com/spf13/viper"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime/debug"
 )
@@ -119,9 +120,10 @@ func (s *SpiderSync) Download() {
 		return
 	}
 
-	//修改目标文件夹权限
+	//递归修改目标文件夹权限
 	// 解决scrapy.setting中开启LOG_ENABLED 和 LOG_FILE时不能创建log文件的问题
-	if err := os.Chmod(dstPath, 0777); err != nil {
+	cmd := exec.Command("chmod", "-R", "777", dstPath)
+	if err := cmd.Run(); err != nil {
 		log.Errorf(err.Error())
 		debug.PrintStack()
 		return

From 0bbcd1b64fb4b3c7823e9dfc1590554eb74ad9fe Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 12:39:07 +0800
Subject: [PATCH 18/52] =?UTF-8?q?fixed:=20=E7=82=B9=E5=87=BB=E5=88=97?=
 =?UTF-8?q?=E8=A1=A8=E6=8C=89=E9=92=AE=E5=AF=BC=E8=88=AA=E8=BF=9B=E5=85=A5?=
 =?UTF-8?q?=E8=AF=A6=E6=83=85=E9=A1=B5=E9=9D=A2=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue |  6 ++---
 frontend/src/views/spider/SpiderList.vue      | 26 +++++++++++--------
 frontend/src/views/task/TaskList.vue          |  5 ++--
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 1b03c335..7b7c7ffd 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -287,14 +287,12 @@ export default {
       return ''
     },
     stageNodes () {
-      const elChart = document.querySelector('#process-chart')
-      const totalWidth = Number(getComputedStyle(elChart).width.replace('px', ''))
+      // const elChart = document.querySelector('#process-chart')
+      // const totalWidth = Number(getComputedStyle(elChart).width.replace('px', ''))
       const stages = Object.values(this.spiderForm.config.stages)
       return stages.map((stage, i) => {
         return {
           name: stage.name,
-          // x: i * totalWidth / stages.length,
-          // y: 0,
           ...stage
         }
       })
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index 22563796..11884a7f 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -221,16 +221,19 @@
         <el-table-column :label="$t('Action')" align="left" fixed="right">
           <template slot-scope="scope">
             <el-tooltip :content="$t('View')" placement="top">
-              <el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
+              <el-button type="primary" icon="el-icon-search" size="mini"
+                         @click="onView(scope.row, $event)"></el-button>
             </el-tooltip>
             <el-tooltip :content="$t('Remove')" placement="top">
-              <el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
+              <el-button type="danger" icon="el-icon-delete" size="mini"
+                         @click="onRemove(scope.row, $event)"></el-button>
             </el-tooltip>
             <el-tooltip v-if="!isShowRun(scope.row)" :content="$t('No command line')" placement="top">
-              <el-button disabled type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
+              <el-button disabled type="success" icon="fa fa-bug" size="mini"
+                         @click="onCrawl(scope.row, $event)"></el-button>
             </el-tooltip>
             <el-tooltip v-else :content="$t('Run')" placement="top">
-              <el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
+              <el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row, $event)"></el-button>
             </el-tooltip>
           </template>
         </el-table-column>
@@ -385,7 +388,8 @@ export default {
       this.$store.commit('spider/SET_SPIDER_FORM', row)
       this.dialogVisible = true
     },
-    onRemove (row) {
+    onRemove (row, ev) {
+      ev.stopPropagation()
       this.$confirm(this.$t('Are you sure to delete this spider?'), this.$t('Notification'), {
         confirmButtonText: this.$t('Confirm'),
         cancelButtonText: this.$t('Cancel'),
@@ -401,12 +405,14 @@ export default {
         this.$st.sendEv('爬虫', '删除')
       })
     },
-    onCrawl (row) {
+    onCrawl (row, ev) {
+      ev.stopPropagation()
       this.crawlConfirmDialogVisible = true
       this.activeSpiderId = row._id
       this.$st.sendEv('爬虫', '点击运行')
     },
-    onView (row) {
+    onView (row, ev) {
+      ev.stopPropagation()
       this.$router.push('/spiders/' + row._id)
       this.$st.sendEv('爬虫', '查看')
     },
@@ -494,10 +500,8 @@ export default {
       if (!str || str.match('^0001')) return 'NA'
       return dayjs(str).format('YYYY-MM-DD HH:mm:ss')
     },
-    onRowClick (row, event, column) {
-      if (column.label !== this.$t('Action')) {
-        this.onView(row)
-      }
+    onRowClick (row, column, event) {
+      this.onView(row, event)
     },
     onClickTab (tab) {
       this.filter.type = tab.name
diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue
index 9db3623d..3ab19fee 100644
--- a/frontend/src/views/task/TaskList.vue
+++ b/frontend/src/views/task/TaskList.vue
@@ -125,7 +125,7 @@
               <el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
             </el-tooltip>
             <el-tooltip :content="$t('Remove')" placement="top">
-              <el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
+              <el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row, $event)"></el-button>
             </el-tooltip>
           </template>
         </el-table-column>
@@ -250,7 +250,8 @@ export default {
     onSelectSpider () {
       this.$st.sendEv('任务', '选择爬虫')
     },
-    onRemove (row) {
+    onRemove (row, ev) {
+      ev.stopPropagation()
       this.$confirm(this.$t('Are you sure to delete this task?'), this.$t('Notification'), {
         confirmButtonText: this.$t('Confirm'),
         cancelButtonText: this.$t('Cancel'),

From b9c7920b766cf0da935dfbf49b15646bdfaaf8b0 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 12:54:56 +0800
Subject: [PATCH 19/52] =?UTF-8?q?fixed=20=E9=85=8D=E7=BD=AE=E4=B8=8B?=
 =?UTF-8?q?=E5=88=92=E7=BA=BF=E4=B8=8D=E6=98=BE=E7=A4=BA=E7=9A=84=E9=97=AE?=
 =?UTF-8?q?=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 10 ++++++++++
 frontend/src/views/spider/SpiderDetail.vue    |  8 ++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 7b7c7ffd..1c127ca3 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -538,6 +538,16 @@ export default {
     },
     onTabClick (tab) {
       this.activeTab = tab.name
+    },
+    update () {
+      if (this.activeTab !== 'stages') return
+
+      // 手动显示tab下划线
+      const elBar = document.querySelector('.el-tabs__active-bar')
+      const elStages = document.querySelector('#tab-stages')
+      const totalWidth = Number(getComputedStyle(elStages).width.replace('px', ''))
+      const paddingRight = Number(getComputedStyle(elStages).paddingRight.replace('px', ''))
+      elBar.setAttribute('style', 'width:' + (totalWidth - paddingRight) + 'px')
     }
   },
   created () {
diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue
index b42e750d..29b531ef 100644
--- a/frontend/src/views/spider/SpiderDetail.vue
+++ b/frontend/src/views/spider/SpiderDetail.vue
@@ -13,8 +13,8 @@
       <el-tab-pane :label="$t('Overview')" name="overview">
         <spider-overview/>
       </el-tab-pane>
-      <el-tab-pane v-if="isConfigurable" :label="$t('Config')" name="配置">
-        <config-list/>
+      <el-tab-pane v-if="isConfigurable" :label="$t('Config')" name="config">
+        <config-list ref="config"/>
       </el-tab-pane>
       <el-tab-pane :label="$t('Files')" name="files">
         <file-list/>
@@ -77,6 +77,10 @@ export default {
         setTimeout(() => {
           this.$refs['spider-stats'].update()
         }, 0)
+      } else if (this.activeTabName === 'config') {
+        setTimeout(() => {
+          this.$refs['config'].update()
+        }, 0)
       }
       this.$st.sendEv('爬虫详情', '切换标签', tab.name)
     },

From d71812147a0bb0ff06eaf4ed7e2124995d93da57 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 13:32:33 +0800
Subject: [PATCH 20/52] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E5=89=8D=E7=AB=AF=E7=95=8C=E9=9D=A2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 27 +++++++++++++++++--
 .../components/TableView/FieldsTableView.vue  | 15 ++++++++++-
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 1c127ca3..e7c7212a 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -493,6 +493,7 @@ export default {
         },
         series: [
           {
+            animation: false,
             type: 'graph',
             layout: 'force',
             symbolSize: 50,
@@ -515,7 +516,7 @@ export default {
             force: {
               initLayout: 'force',
               repulsion: 100,
-              gravity: 0.01,
+              gravity: 0.00001,
               edgeLength: 200
             },
             // draggable: true,
@@ -529,7 +530,29 @@ export default {
               }
             }
           }
-        ]
+        ],
+        tooltip: {
+          // formatter: '{b0}: {c0}<br />{b1}: {c1}',
+          formatter: (params) => {
+            if (!params.data.fields) return
+
+            let str = ''
+            str += `<label>${this.$t('Stage')}: ${params.name}</label><br><br>`
+            str += `<label>${this.$t('Fields')}: </label><br>`
+            str += '<ul style="list-style: none; padding: 0; margin: 0;">'
+            for (let i = 0; i < params.data.fields.length; i++) {
+              const f = params.data.fields[i]
+              str += `
+<li>
+<span style="display: inline-block; min-width: 50px; font-weight: bolder; text-align: right">${f.name}: </span>
+${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --> ' + '<span style="font-weight:bolder">' + f.next_stage + '</span>') : ''}
+</li>
+`
+            }
+            str += '</ul>'
+            return str
+          }
+        }
       }
       const el = document.querySelector('#process-chart')
       this.processChart = echarts.init(el)
diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue
index 220deda8..c9f25c89 100644
--- a/frontend/src/components/TableView/FieldsTableView.vue
+++ b/frontend/src/components/TableView/FieldsTableView.vue
@@ -83,7 +83,7 @@
               <el-input v-model="scope.row.attr" :placeholder="$t('Attribute')"/>
             </template>
             <template v-else>
-              <span style="margin-left: 15px">
+              <span style="margin-left: 15px; color: lightgrey">
                 N/A
               </span>
             </template>
@@ -93,6 +93,8 @@
           <template slot-scope="scope">
             <el-select
               v-model="scope.row.next_stage"
+              :class="!scope.row.next_stage ? 'disabled' : ''"
+              @change="onChangeNextStage(scope.row)"
             >
               <el-option :label="$t('No Next Stage')" value=""/>
               <el-option v-for="n in stageNames" :key="n" :label="n" :value="n"/>
@@ -237,6 +239,13 @@ export default {
           }
         }
       }
+    },
+    onChangeNextStage (row) {
+      this.fields.forEach(f => {
+        if (f.name !== row.name) {
+          this.$set(f, 'next_stage', '')
+        }
+      })
     }
   }
 }
@@ -333,4 +342,8 @@ export default {
   .action-button-group >>> .el-checkbox__label {
     font-size: 12px;
   }
+
+  .el-table.edit >>> .el-select.disabled .el-input__inner {
+    color: lightgrey;
+  }
 </style>

From 8210fa6c20becfee373b485ea474848d9a5615d6 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 16:08:13 +0800
Subject: [PATCH 21/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E4=BF=AE=E6=94=B9?=
 =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AEPOST=E6=8E=A5=E5=8F=A3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/main.go                       |   9 ++-
 backend/model/config_spider/scrapy.go |   8 +-
 backend/routes/config_spider.go       | 108 +++++++-------------------
 backend/services/config_spider.go     |  93 ++++++++++++++++++++++
 4 files changed, 128 insertions(+), 90 deletions(-)

diff --git a/backend/main.go b/backend/main.go
index 436c537e..72ab2c25 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -140,10 +140,11 @@ func main() {
 			authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats)   // 爬虫统计数据
 			authGroup.GET("/spider/types", routes.GetSpiderTypes)        // 爬虫类型
 			// 可配置爬虫
-			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 可配置爬虫配置
-			authGroup.PUT("/config_spiders", routes.PutConfigSpider)                  // 添加可配置爬虫
-			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)            // 修改可配置爬虫
-			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider)   // 上传可配置爬虫
+			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig)   // 获取可配置爬虫配置
+			authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
+			authGroup.PUT("/config_spiders", routes.PutConfigSpider)                    // 添加可配置爬虫
+			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)              // 修改可配置爬虫
+			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider)     // 上传可配置爬虫
 			// 任务
 			authGroup.GET("/tasks", routes.GetTaskList)                                 // 任务列表
 			authGroup.GET("/tasks/:id", routes.GetTask)                                 // 任务详情
diff --git a/backend/model/config_spider/scrapy.go b/backend/model/config_spider/scrapy.go
index 9da6fb89..62c5cd78 100644
--- a/backend/model/config_spider/scrapy.go
+++ b/backend/model/config_spider/scrapy.go
@@ -216,19 +216,19 @@ func (g ScrapyGenerator) GetExtractStringFromField(f entity.Field) string {
 		// 如果为CSS
 		if f.Attr == "" {
 			// 文本
-			return fmt.Sprintf(`css(%s::text())`, f.Css)
+			return fmt.Sprintf(`css('%s::text()')`, f.Css)
 		} else {
 			// 属性
-			return fmt.Sprintf(`css(%s::attr("%s"))`, f.Css, f.Attr)
+			return fmt.Sprintf(`css('%s::attr("%s")')`, f.Css, f.Attr)
 		}
 	} else {
 		// 如果为XPath
 		if f.Attr == "" {
 			// 文本
-			return fmt.Sprintf(`xpath(%s/text())`, f.Xpath)
+			return fmt.Sprintf(`xpath('%s/text()')`, f.Xpath)
 		} else {
 			// 属性
-			return fmt.Sprintf(`xpath(%s/@%s)`, f.Xpath, f.Attr)
+			return fmt.Sprintf(`xpath('%s/@%s')`, f.Xpath, f.Attr)
 		}
 	}
 }
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index cdd40607..1d10335e 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -2,16 +2,13 @@ package routes
 
 import (
 	"crawlab/constants"
-	"crawlab/database"
 	"crawlab/entity"
 	"crawlab/model"
 	"crawlab/services"
 	"crawlab/utils"
 	"fmt"
-	"github.com/apex/log"
 	"github.com/gin-gonic/gin"
 	"github.com/globalsign/mgo/bson"
-	uuid "github.com/satori/go.uuid"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
 	"io"
@@ -19,7 +16,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"runtime/debug"
 )
 
 // 添加可配置爬虫
@@ -151,91 +147,39 @@ func UploadConfigSpider(c *gin.Context) {
 		return
 	}
 
-	// 赋值 stage_name
-	for stageName, stage := range configData.Stages {
-		stage.Name = stageName
-		configData.Stages[stageName] = stage
-	}
-
-	// 删除已有的爬虫文件
-	for _, fInfo := range utils.ListDir(spiderDir) {
-		// 不删除Spiderfile
-		if fInfo.Name() == filename {
-			continue
-		}
-
-		// 删除其他文件
-		if err := os.RemoveAll(filepath.Join(spiderDir, fInfo.Name())); err != nil {
-			HandleError(http.StatusInternalServerError, c, err)
-			return
-		}
-	}
-
-	// 拷贝爬虫文件
-	tplDir := "./template/scrapy"
-	for _, fInfo := range utils.ListDir(tplDir) {
-		// 跳过Spiderfile
-		if fInfo.Name() == "Spiderfile" {
-			continue
-		}
-
-		srcPath := filepath.Join(tplDir, fInfo.Name())
-		if fInfo.IsDir() {
-			dirPath := filepath.Join(spiderDir, fInfo.Name())
-			if err := utils.CopyDir(srcPath, dirPath); err != nil {
-				HandleError(http.StatusInternalServerError, c, err)
-				return
-			}
-		} else {
-			if err := utils.CopyFile(srcPath, filepath.Join(spiderDir, fInfo.Name())); err != nil {
-				HandleError(http.StatusInternalServerError, c, err)
-				return
-			}
-		}
-	}
-
-	// 更改爬虫文件
-	if err := services.GenerateConfigSpiderFiles(spider, configData); err != nil {
+	// 根据序列化后的数据处理爬虫文件
+	if err := services.ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
-		return
 	}
 
-	// 打包为 zip 文件
-	files, err := utils.GetFilesFromDir(spiderDir)
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func PostConfigSpiderConfig(c *gin.Context) {
+	id := c.Param("id")
+
+	// 获取爬虫
+	var spider model.Spider
+	spider, err := model.GetSpider(bson.ObjectIdHex(id))
 	if err != nil {
+		HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("cannot find spider (id: %s)", id))
+	}
+
+	// 反序列化配置数据
+	var configData entity.ConfigSpiderData
+	if err := c.ShouldBindJSON(&configData); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+	}
+
+	// 根据序列化后的数据处理爬虫文件
+	if err := services.ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
-		return
-	}
-	randomId := uuid.NewV4()
-	tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), spider.Name+"."+randomId.String()+".zip")
-	spiderZipFileName := spider.Name + ".zip"
-	if err := utils.Compress(files, tmpFilePath); err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
-		return
 	}
 
-	// 获取 GridFS 实例
-	s, gf := database.GetGridFs("files")
-	defer s.Close()
-
-	// 判断文件是否已经存在
-	var gfFile model.GridFs
-	if err := gf.Find(bson.M{"filename": spiderZipFileName}).One(&gfFile); err == nil {
-		// 已经存在文件，则删除
-		_ = gf.RemoveId(gfFile.Id)
-	}
-
-	// 上传到GridFs
-	fid, err := services.UploadToGridFs(spiderZipFileName, tmpFilePath)
-	if err != nil {
-		log.Errorf("upload to grid fs error: %s", err.Error())
-		debug.PrintStack()
-		return
-	}
-
-	// 保存爬虫 FileId
-	spider.FileId = fid
-	_ = spider.Save()
+	// TODO: 替换Spiderfile文件
 
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go
index 35bb2790..2d9acaba 100644
--- a/backend/services/config_spider.go
+++ b/backend/services/config_spider.go
@@ -2,11 +2,19 @@ package services
 
 import (
 	"crawlab/constants"
+	"crawlab/database"
 	"crawlab/entity"
 	"crawlab/model"
 	"crawlab/model/config_spider"
+	"crawlab/utils"
 	"errors"
 	"fmt"
+	"github.com/apex/log"
+	"github.com/globalsign/mgo/bson"
+	uuid "github.com/satori/go.uuid"
+	"github.com/spf13/viper"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -139,3 +147,88 @@ func IsUniqueConfigSpiderFields(fields []entity.Field) bool {
 	}
 	return true
 }
+
+func ProcessSpiderFilesFromConfigData(spider model.Spider, configData entity.ConfigSpiderData) error {
+	spiderDir := spider.Src
+
+	// 赋值 stage_name
+	for stageName, stage := range configData.Stages {
+		stage.Name = stageName
+		configData.Stages[stageName] = stage
+	}
+
+	// 删除已有的爬虫文件
+	for _, fInfo := range utils.ListDir(spiderDir) {
+		// 不删除Spiderfile
+		if fInfo.Name() == "Spiderfile" {
+			continue
+		}
+
+		// 删除其他文件
+		if err := os.RemoveAll(filepath.Join(spiderDir, fInfo.Name())); err != nil {
+			return err
+		}
+	}
+
+	// 拷贝爬虫文件
+	tplDir := "./template/scrapy"
+	for _, fInfo := range utils.ListDir(tplDir) {
+		// 跳过Spiderfile
+		if fInfo.Name() == "Spiderfile" {
+			continue
+		}
+
+		srcPath := filepath.Join(tplDir, fInfo.Name())
+		if fInfo.IsDir() {
+			dirPath := filepath.Join(spiderDir, fInfo.Name())
+			if err := utils.CopyDir(srcPath, dirPath); err != nil {
+				return err
+			}
+		} else {
+			if err := utils.CopyFile(srcPath, filepath.Join(spiderDir, fInfo.Name())); err != nil {
+				return err
+			}
+		}
+	}
+
+	// 更改爬虫文件
+	if err := GenerateConfigSpiderFiles(spider, configData); err != nil {
+		return err
+	}
+
+	// 打包为 zip 文件
+	files, err := utils.GetFilesFromDir(spiderDir)
+	if err != nil {
+		return err
+	}
+	randomId := uuid.NewV4()
+	tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), spider.Name+"."+randomId.String()+".zip")
+	spiderZipFileName := spider.Name + ".zip"
+	if err := utils.Compress(files, tmpFilePath); err != nil {
+		return err
+	}
+
+	// 获取 GridFS 实例
+	s, gf := database.GetGridFs("files")
+	defer s.Close()
+
+	// 判断文件是否已经存在
+	var gfFile model.GridFs
+	if err := gf.Find(bson.M{"filename": spiderZipFileName}).One(&gfFile); err == nil {
+		// 已经存在文件，则删除
+		_ = gf.RemoveId(gfFile.Id)
+	}
+
+	// 上传到GridFs
+	fid, err := UploadToGridFs(spiderZipFileName, tmpFilePath)
+	if err != nil {
+		log.Errorf("upload to grid fs error: %s", err.Error())
+		return err
+	}
+
+	// 保存爬虫 FileId
+	spider.FileId = fid
+	_ = spider.Save()
+
+	return nil
+}

From d65e37517e7bc67b8beaf25324f43062591eaa83 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 21:36:17 +0800
Subject: [PATCH 22/52] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E5=89=8D=E7=AB=AF=E7=95=8C=E9=9D=A2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/main.go                               |  11 +-
 backend/model/config_spider/scrapy.go         |   6 +-
 backend/routes/config_spider.go               |  57 ++++++-
 backend/services/config_spider.go             |  26 ++++
 frontend/src/components/Config/ConfigList.vue | 140 +++++++++---------
 frontend/src/components/File/FileDetail.vue   |  10 +-
 frontend/src/components/File/FileList.vue     |   2 +
 .../components/InfoView/SpiderInfoView.vue    |   6 +-
 frontend/src/store/modules/file.js            |   7 +-
 frontend/src/store/modules/spider.js          |   7 +
 frontend/src/views/spider/SpiderDetail.vue    |   7 +
 11 files changed, 186 insertions(+), 93 deletions(-)

diff --git a/backend/main.go b/backend/main.go
index 72ab2c25..226b3bd5 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -140,11 +140,12 @@ func main() {
 			authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats)   // 爬虫统计数据
 			authGroup.GET("/spider/types", routes.GetSpiderTypes)        // 爬虫类型
 			// 可配置爬虫
-			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig)   // 获取可配置爬虫配置
-			authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
-			authGroup.PUT("/config_spiders", routes.PutConfigSpider)                    // 添加可配置爬虫
-			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)              // 修改可配置爬虫
-			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider)     // 上传可配置爬虫
+			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig)           // 获取可配置爬虫配置
+			authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig)         // 更改可配置爬虫配置
+			authGroup.PUT("/config_spiders", routes.PutConfigSpider)                            // 添加可配置爬虫
+			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)                      // 修改可配置爬虫
+			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider)             // 上传可配置爬虫
+			authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
 			// 任务
 			authGroup.GET("/tasks", routes.GetTaskList)                                 // 任务列表
 			authGroup.GET("/tasks/:id", routes.GetTask)                                 // 任务详情
diff --git a/backend/model/config_spider/scrapy.go b/backend/model/config_spider/scrapy.go
index 62c5cd78..5a6730d1 100644
--- a/backend/model/config_spider/scrapy.go
+++ b/backend/model/config_spider/scrapy.go
@@ -216,7 +216,7 @@ func (g ScrapyGenerator) GetExtractStringFromField(f entity.Field) string {
 		// 如果为CSS
 		if f.Attr == "" {
 			// 文本
-			return fmt.Sprintf(`css('%s::text()')`, f.Css)
+			return fmt.Sprintf(`css('%s::text')`, f.Css)
 		} else {
 			// 属性
 			return fmt.Sprintf(`css('%s::attr("%s")')`, f.Css, f.Attr)
@@ -242,9 +242,9 @@ func (g ScrapyGenerator) GetExtractStringFromStage(stage entity.Stage) string {
 
 	if stage.PageCss != "" {
 		// 如果为CSS
-		return fmt.Sprintf(`css(%s::attr("%s"))`, stage.PageCss, pageAttr)
+		return fmt.Sprintf(`css('%s::attr("%s")')`, stage.PageCss, pageAttr)
 	} else {
 		// 如果为XPath
-		return fmt.Sprintf(`xpath(%s/@%s)`, stage.PageXpath, pageAttr)
+		return fmt.Sprintf(`xpath('%s/@%s')`, stage.PageXpath, pageAttr)
 	}
 }
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 1d10335e..68998737 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -158,6 +158,54 @@ func UploadConfigSpider(c *gin.Context) {
 	})
 }
 
+func PostConfigSpiderSpiderfile(c *gin.Context) {
+	type Body struct {
+		Content string `json:"content"`
+	}
+
+	id := c.Param("id")
+
+	// 文件内容
+	var reqBody Body
+	if err := c.ShouldBindJSON(&reqBody); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+	content := reqBody.Content
+
+	// 获取爬虫
+	var spider model.Spider
+	spider, err := model.GetSpider(bson.ObjectIdHex(id))
+	if err != nil {
+		HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("cannot find spider (id: %s)", id))
+		return
+	}
+
+	// 反序列化
+	var configData entity.ConfigSpiderData
+	if err := yaml.Unmarshal([]byte(content), &configData); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+
+	// 根据序列化后的数据处理爬虫文件
+	if err := services.ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 写文件
+	if err := ioutil.WriteFile(filepath.Join(spider.Src, "Spiderfile"), []byte(content), os.ModePerm); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
 func PostConfigSpiderConfig(c *gin.Context) {
 	id := c.Param("id")
 
@@ -166,20 +214,27 @@ func PostConfigSpiderConfig(c *gin.Context) {
 	spider, err := model.GetSpider(bson.ObjectIdHex(id))
 	if err != nil {
 		HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("cannot find spider (id: %s)", id))
+		return
 	}
 
 	// 反序列化配置数据
 	var configData entity.ConfigSpiderData
 	if err := c.ShouldBindJSON(&configData); err != nil {
 		HandleError(http.StatusBadRequest, c, err)
+		return
 	}
 
 	// 根据序列化后的数据处理爬虫文件
 	if err := services.ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
+		return
 	}
 
-	// TODO: 替换Spiderfile文件
+	// 替换Spiderfile文件
+	if err := services.GenerateSpiderfileFromConfigData(spider, configData); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
 
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go
index 2d9acaba..d96146f5 100644
--- a/backend/services/config_spider.go
+++ b/backend/services/config_spider.go
@@ -13,6 +13,7 @@ import (
 	"github.com/globalsign/mgo/bson"
 	uuid "github.com/satori/go.uuid"
 	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
 	"os"
 	"path/filepath"
 	"strings"
@@ -232,3 +233,28 @@ func ProcessSpiderFilesFromConfigData(spider model.Spider, configData entity.Con
 
 	return nil
 }
+
+func GenerateSpiderfileFromConfigData(spider model.Spider, configData entity.ConfigSpiderData) error {
+	// Spiderfile 路径
+	sfPath := filepath.Join(spider.Src, "Spiderfile")
+
+	// 生成Yaml内容
+	sfContentByte, err := yaml.Marshal(configData)
+	if err != nil {
+		return err
+	}
+
+	// 打开文件
+	f, err := os.OpenFile(sfPath, os.O_WRONLY|os.O_TRUNC, 0777)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// 写入内容
+	if _, err := f.Write(sfContentByte); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index e7c7212a..014bfd8d 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -120,10 +120,10 @@
           <div class="button-group-container">
             <div class="button-group">
               <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
-              <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">
-                {{$t('ExtractFields')}}
-              </el-button>
-              <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
+              <!--              <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">-->
+              <!--                {{$t('ExtractFields')}}-->
+              <!--              </el-button>-->
+              <!--              <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>-->
               <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
             </div>
           </div>
@@ -181,6 +181,18 @@
         <div id="process-chart"></div>
       </el-tab-pane>
       <!--./Graph-->
+
+      <!--Spiderfile-->
+      <el-tab-pane name="spiderfile" label="Spiderfile">
+        <div class="spiderfile-actions">
+          <el-button type="primary" size="small" style="margin-right: 10px;" @click="onSpiderfileSave">
+            <font-awesome-icon :icon="['fa', 'save']"/>
+            {{$t('Save')}}
+          </el-button>
+        </div>
+        <file-detail/>
+      </el-tab-pane>
+      <!--./Spiderfile-->
     </el-tabs>
     <!--./tabs-->
   </div>
@@ -194,16 +206,25 @@ import echarts from 'echarts'
 import FieldsTableView from '../TableView/FieldsTableView'
 import CrawlConfirmDialog from '../Common/CrawlConfirmDialog'
 
+import 'codemirror/lib/codemirror.js'
+import 'codemirror/mode/yaml/yaml.js'
+import FileDetail from '../File/FileDetail'
+
 export default {
   name: 'ConfigList',
   components: {
+    FileDetail,
     CrawlConfirmDialog,
     FieldsTableView
   },
   watch: {
     activeTab () {
       setTimeout(() => {
+        // 渲染流程图
         this.renderProcessChart()
+
+        // 获取Spiderfile
+        this.getSpiderfile()
       }, 0)
     }
   },
@@ -229,7 +250,15 @@ export default {
         { name: 'next_stage', label: 'Next Stage' }
       ],
       activeTab: 'stages',
-      processChart: undefined
+      processChart: undefined,
+      fileOptions: {
+        mode: 'text/x-yaml',
+        theme: 'darcula',
+        styleActiveLine: true,
+        lineNumbers: true,
+        line: true,
+        matchBrackets: true
+      }
     }
   },
   computed: {
@@ -318,34 +347,16 @@ export default {
     onSelectCrawlType (value) {
       this.spiderForm.crawl_type = value
     },
-    onSave () {
+    async onSave () {
       this.$st.sendEv('爬虫详情-配置', '保存')
-      return new Promise((resolve, reject) => {
-        this.saveLoading = true
-        this.$store.dispatch('spider/updateSpiderFields')
-          .then(() => {
-            this.$store.dispatch('spider/editSpider')
-              .then(() => {
-                this.$message.success(this.$t('Spider info has been saved successfully'))
-                resolve()
-              })
-              .catch(() => {
-                this.$message.error(this.$t('Something wrong happened'))
-                reject(new Error())
-              })
-              .finally(() => {
-                this.saveLoading = false
-              })
-          })
-          .then(() => {
-            this.$store.dispatch('spider/updateSpiderDetailFields')
-          })
-          .catch(() => {
-            this.$message.error(this.$t('Something wrong happened'))
-            this.saveLoading = false
-            reject(new Error())
-          })
-      })
+      this.saveLoading = true
+      try {
+        await this.$store.dispatch('spider/postConfigSpiderConfig')
+        this.$message.success(this.$t('Spider info has been saved successfully'))
+      } catch (e) {
+        this.$message.error(this.$t('Something wrong happened'))
+      }
+      this.saveLoading = false
     },
     onDialogClose () {
       this.dialogVisible = false
@@ -378,17 +389,8 @@ export default {
       })
     },
     onCrawl () {
-      this.$confirm(this.$t('Are you sure to run this spider?'), this.$t('Notification'), {
-        confirmButtonText: this.$t('Confirm'),
-        cancelButtonText: this.$t('Cancel')
-      })
-        .then(() => {
-          this.$store.dispatch('spider/crawlSpider', this.spiderForm._id)
-            .then(() => {
-              this.$message.success(this.$t(`Spider task has been scheduled`))
-            })
-          this.$st.sendEv('爬虫详情-配置', '运行')
-        })
+      this.crawlConfirmDialogVisible = true
+      this.$st.sendEv('爬虫详情-配置', '点击运行')
     },
     onExtractFields () {
       this.$refs['form'].validate(res => {
@@ -571,39 +573,20 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
       const totalWidth = Number(getComputedStyle(elStages).width.replace('px', ''))
       const paddingRight = Number(getComputedStyle(elStages).paddingRight.replace('px', ''))
       elBar.setAttribute('style', 'width:' + (totalWidth - paddingRight) + 'px')
-    }
-  },
-  created () {
-    // fields for list page
-    if (!this.spiderForm.fields) {
-      this.spiderForm.fields = []
-      for (let i = 0; i < 3; i++) {
-        this.spiderForm.fields.push({
-          name: 'field_' + (i + 1),
-          type: 'css',
-          extract_type: 'text'
-        })
+    },
+    getSpiderfile () {
+      this.$store.commit('file/SET_FILE_CONTENT', '')
+      this.$store.commit('file/SET_CURRENT_PATH', 'Spiderfile')
+      this.$store.dispatch('file/getFileContent', { path: 'Spiderfile' })
+    },
+    async onSpiderfileSave () {
+      try {
+        await this.$store.dispatch('spider/saveConfigSpiderSpiderfile')
+        this.$message.success(this.$t('Spiderfile saved successfully'))
+      } catch (e) {
+        this.$message.error('Something wrong happened')
       }
     }
-
-    // fields for detail page
-    if (!this.spiderForm.detail_fields) {
-      this.spiderForm.detail_fields = []
-      for (let i = 0; i < 3; i++) {
-        this.spiderForm.detail_fields.push({
-          name: 'field_' + (i + 1),
-          type: 'css',
-          extract_type: 'text'
-        })
-      }
-    }
-
-    if (!this.spiderForm.crawl_type) this.$set(this.spiderForm, 'crawl_type', 'list')
-    // if (!this.spiderForm.start_url) this.$set(this.spiderForm, 'start_url', 'http://example.com')
-    if (!this.spiderForm.item_selector_type) this.$set(this.spiderForm, 'item_selector_type', 'css')
-    if (!this.spiderForm.pagination_selector_type) this.$set(this.spiderForm, 'pagination_selector_type', 'css')
-    if (this.spiderForm.obey_robots_txt == null) this.$set(this.spiderForm, 'obey_robots_txt', true)
-    if (this.spiderForm.item_threshold == null) this.$set(this.spiderForm, 'item_threshold', 10)
   },
   mounted () {
     this.activeNames = Object.keys(this.spiderForm.config.stages)
@@ -770,4 +753,13 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
     width: 100%;
     height: 480px;
   }
+
+  .config-list >>> .file-content {
+    height: calc(100vh - 280px);
+  }
+
+  .spiderfile-actions {
+    margin-bottom: 5px;
+    text-align: right;
+  }
 </style>
diff --git a/frontend/src/components/File/FileDetail.vue b/frontend/src/components/File/FileDetail.vue
index d74f73b3..f5f8a4cc 100644
--- a/frontend/src/components/File/FileDetail.vue
+++ b/frontend/src/components/File/FileDetail.vue
@@ -18,6 +18,7 @@ import 'codemirror/mode/go/go.js'
 import 'codemirror/mode/shell/shell.js'
 import 'codemirror/mode/markdown/markdown.js'
 import 'codemirror/mode/php/php.js'
+import 'codemirror/mode/yaml/yaml.js'
 
 export default {
   name: 'FileDetail',
@@ -38,7 +39,7 @@ export default {
     },
     options () {
       return {
-        mode: this.lanaguage,
+        mode: this.language,
         theme: 'darcula',
         styleActiveLine: true,
         lineNumbers: true,
@@ -46,8 +47,9 @@ export default {
         matchBrackets: true
       }
     },
-    lanaguage () {
+    language () {
       const fileName = this.$store.state.file.currentPath
+      if (!fileName) return ''
       if (fileName.match(/\.js$/)) {
         return 'text/javascript'
       } else if (fileName.match(/\.py$/)) {
@@ -60,6 +62,8 @@ export default {
         return 'text/x-php'
       } else if (fileName.match(/\.md$/)) {
         return 'text/x-markdown'
+      } else if (fileName === 'Spiderfile') {
+        return 'text/x-yaml'
       } else {
         return 'text'
       }
@@ -74,7 +78,7 @@ export default {
 <style scoped>
   .file-content {
     border: 1px solid #eaecef;
-    height: 480px;
+    height: calc(100vh - 256px);
   }
 
   .file-content >>> .CodeMirror {
diff --git a/frontend/src/components/File/FileList.vue b/frontend/src/components/File/FileList.vue
index 4d263a7a..e874c446 100644
--- a/frontend/src/components/File/FileList.vue
+++ b/frontend/src/components/File/FileList.vue
@@ -120,6 +120,8 @@ export default {
       this.showFile = false
       this.onBack()
     }
+  },
+  created () {
   }
 }
 </script>
diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue
index 50f1ccf8..bf05a2ba 100644
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -102,7 +102,11 @@ export default {
       'spiderForm'
     ]),
     isShowRun () {
-      return !!this.spiderForm.cmd
+      if (this.spiderForm.type === 'customized') {
+        return !!this.spiderForm.cmd
+      } else {
+        return true
+      }
     }
   },
   methods: {
diff --git a/frontend/src/store/modules/file.js b/frontend/src/store/modules/file.js
index 5cc50acb..107a82d5 100644
--- a/frontend/src/store/modules/file.js
+++ b/frontend/src/store/modules/file.js
@@ -43,12 +43,7 @@ const actions = {
         commit('SET_FILE_CONTENT', response.data.data)
       })
   },
-  saveFileContent ({ state, rootState }, payload) {
-    const { path } = payload
-    const spiderId = rootState.spider.spiderForm._id
-    const content = state.fileContent
-    return request.post(`/spiders/${spiderId}/file`, { content, path })
-  }
+
 }
 
 export default {
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index 07a0bac3..b7093da4 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -148,6 +148,13 @@ const actions = {
   },
   extractFields ({ state, commit }) {
     return request.post(`/spiders/${state.spiderForm._id}/extract_fields`)
+  },
+  postConfigSpiderConfig ({ state }) {
+    return request.post(`/config_spiders/${state.spiderForm._id}/config`, state.spiderForm.config)
+  },
+  saveConfigSpiderSpiderfile ({ state, rootState }) {
+    const content = rootState.file.fileContent
+    return request.post(`/config_spiders/${state.spiderForm._id}/spiderfile`, { content })
   }
 }
 
diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue
index 29b531ef..4986957a 100644
--- a/frontend/src/views/spider/SpiderDetail.vue
+++ b/frontend/src/views/spider/SpiderDetail.vue
@@ -48,6 +48,13 @@ export default {
     FileList,
     SpiderOverview
   },
+  watch: {
+    activeTabName () {
+      // 初始化文件
+      this.$store.commit('file/SET_FILE_CONTENT', '')
+      this.$store.commit('file/SET_CURRENT_PATH', '')
+    }
+  },
   data () {
     return {
       activeTabName: 'overview'

From ee0ad4cede55e3f98ad483f1428caf8842ec5875 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 22:19:49 +0800
Subject: [PATCH 23/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=B7=BB=E5=8A=A0?=
 =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC=E8=99=AB?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/store/modules/spider.js     |   3 +
 frontend/src/views/spider/SpiderList.vue | 102 +++++++++++++++--------
 2 files changed, 72 insertions(+), 33 deletions(-)

diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index b7093da4..e653db3d 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -155,6 +155,9 @@ const actions = {
   saveConfigSpiderSpiderfile ({ state, rootState }) {
     const content = rootState.file.fileContent
     return request.post(`/config_spiders/${state.spiderForm._id}/spiderfile`, { content })
+  },
+  addConfigSpider ({ state }) {
+    return request.put(`/config_spiders`, state.spiderForm)
   }
 }
 
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index 11884a7f..41007044 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -33,18 +33,52 @@
                width="40%"
                :visible.sync="addDialogVisible"
                :before-close="onAddDialogClose">
-      <div class="add-spider-wrapper">
-        <div @click="onAddConfigurable">
-          <el-card shadow="hover" class="add-spider-item success">
-            {{$t('Configurable Spider')}}
-          </el-card>
-        </div>
-        <div @click="onAddCustomized">
-          <el-card shadow="hover" class="add-spider-item primary">
-            {{$t('Customized Spider')}}
-          </el-card>
-        </div>
-      </div>
+      <el-tabs :active-name="spiderType">
+        <el-tab-pane name="customized" :label="$t('Customized')">
+          <el-form :model="spiderForm" ref="addCustomizedForm" inline-message>
+            <el-form-item :label="$t('Upload Zip File')" label-width="120px" name="site">
+              <el-upload
+                :action="$request.baseUrl + '/spiders'"
+                :headers="{Authorization:token}"
+                :on-change="onUploadChange"
+                :on-success="onUploadSuccess"
+                :file-list="fileList">
+                <el-button size="small" type="primary" icon="el-icon-upload">{{$t('Upload')}}</el-button>
+              </el-upload>
+            </el-form-item>
+          </el-form>
+          <el-alert type="error" :title="$t('Please zip your spider files from the root directory')"
+                    :closable="false"></el-alert>
+        </el-tab-pane>
+        <el-tab-pane name="configurable" :label="$t('Configurable')">
+          <el-form :model="spiderForm" ref="addConfigurableForm" inline-message label-width="120px">
+            <el-form-item :label="$t('Spider Name')" prop="name" required>
+              <el-input v-model="spiderForm.name" :placeholder="$t('Spider Name')"/>
+            </el-form-item>
+            <el-form-item :label="$t('Display Name')" prop="display_name" required>
+              <el-input v-model="spiderForm.display_name" :placeholder="$t('Display Name')"/>
+            </el-form-item>
+            <el-form-item :label="$t('Results')" prop="col" required>
+              <el-input v-model="spiderForm.col" :placeholder="$t('Results')"/>
+            </el-form-item>
+          </el-form>
+          <div class="actions">
+            <el-button type="primary" @click="onAddConfigurable">{{$t('Add')}}</el-button>
+          </div>
+        </el-tab-pane>
+      </el-tabs>
+      <!--      <div class="add-spider-wrapper">-->
+      <!--        <div @click="onAddConfigurable">-->
+      <!--          <el-card shadow="hover" class="add-spider-item success">-->
+      <!--            {{$t('Configurable Spider')}}-->
+      <!--          </el-card>-->
+      <!--        </div>-->
+      <!--        <div @click="onAddCustomized">-->
+      <!--          <el-card shadow="hover" class="add-spider-item primary">-->
+      <!--            {{$t('Customized Spider')}}-->
+      <!--          </el-card>-->
+      <!--        </div>-->
+      <!--      </div>-->
     </el-dialog>
     <!--./add dialog-->
 
@@ -81,20 +115,7 @@
                width="40%"
                :visible.sync="addCustomizedDialogVisible"
                :before-close="onAddCustomizedDialogClose">
-      <el-form :model="spiderForm" ref="addConfigurableForm" inline-message>
-        <el-form-item :label="$t('Upload Zip File')" label-width="120px" name="site">
-          <el-upload
-            :action="$request.baseUrl + '/spiders'"
-            :headers="{Authorization:token}"
-            :on-change="onUploadChange"
-            :on-success="onUploadSuccess"
-            :file-list="fileList">
-            <el-button size="small" type="primary" icon="el-icon-upload">{{$t('Upload')}}</el-button>
-          </el-upload>
-        </el-form-item>
-      </el-form>
-      <el-alert type="error" :title="$t('Please zip your spider files from the root directory')"
-                :closable="false"></el-alert>
+
     </el-dialog>
     <!--./customized spider dialog-->
 
@@ -301,7 +322,8 @@ export default {
       spiderFormRules: {
         name: [{ required: true, message: 'Required Field', trigger: 'change' }]
       },
-      fileList: []
+      fileList: [],
+      spiderType: 'customized'
     }
   },
   computed: {
@@ -332,14 +354,24 @@ export default {
       this.getList()
     },
     onAdd () {
-      // this.addDialogVisible = true
-      this.onAddCustomized()
+      this.$store.commit('spider/SET_SPIDER_FORM', {})
+      this.addDialogVisible = true
+      // this.onAddCustomized()
     },
     onAddConfigurable () {
-      this.$store.commit('spider/SET_SPIDER_FORM', {})
-      this.addDialogVisible = false
-      this.addConfigurableDialogVisible = true
-      this.$st.sendEv('爬虫', '添加爬虫-可配置爬虫')
+      this.$refs['addConfigurableForm'].validate(async res => {
+        if (!res) return
+
+        let res2
+        try {
+          res2 = await this.$store.dispatch('spider/addConfigSpider')
+        } catch (e) {
+          this.$message.error(this.$t('Something wrong happened'))
+          return
+        }
+        this.$router.push(`/spiders/${res2.data.data._id}`)
+        this.$st.sendEv('爬虫', '添加爬虫-可配置爬虫')
+      })
     },
     onAddCustomized () {
       this.addDialogVisible = false
@@ -613,4 +645,8 @@ export default {
   .el-table >>> tr {
     cursor: pointer;
   }
+
+  .actions {
+    text-align: right;
+  }
 </style>

From 4ae0043f8eab84796fb9da19fa2eed3c1b6ee57b Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 22:20:20 +0800
Subject: [PATCH 24/52] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=A0=E9=99=A4?=
 =?UTF-8?q?=E5=8F=AF=E9=85=8D=E7=BD=AE=E7=88=AC=E8=99=AB=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/spider.go         | 13 ++++++++-----
 backend/routes/config_spider.go | 17 +++++++++++++++++
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/backend/model/spider.go b/backend/model/spider.go
index 1fe0617f..a06e682b 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -4,6 +4,7 @@ import (
 	"crawlab/constants"
 	"crawlab/database"
 	"crawlab/entity"
+	"crawlab/utils"
 	"errors"
 	"github.com/apex/log"
 	"github.com/globalsign/mgo"
@@ -182,7 +183,7 @@ func GetSpider(id bson.ObjectId) (Spider, error) {
 	}
 
 	// 如果为可配置爬虫，获取爬虫配置
-	if spider.Type == constants.Configurable {
+	if spider.Type == constants.Configurable && utils.Exists(filepath.Join(spider.Src, "Spiderfile")) {
 		config, err := GetConfigSpiderData(spider)
 		if err != nil {
 			return spider, err
@@ -229,10 +230,12 @@ func RemoveSpider(id bson.ObjectId) error {
 	s, gf := database.GetGridFs("files")
 	defer s.Close()
 
-	if err := gf.RemoveId(result.FileId); err != nil {
-		log.Error("remove file error, id:" + result.FileId.Hex())
-		debug.PrintStack()
-		return err
+	if result.FileId.Hex() != constants.ObjectIdNull {
+		if err := gf.RemoveId(result.FileId); err != nil {
+			log.Error("remove file error, id:" + result.FileId.Hex())
+			debug.PrintStack()
+			return err
+		}
 	}
 
 	return nil
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 68998737..982af28d 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -58,6 +58,23 @@ func PutConfigSpider(c *gin.Context) {
 	}
 	spider.Src = spiderDir
 
+	// 复制Spiderfile模版
+	contentByte, err := ioutil.ReadFile("./template/Spiderfile")
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+	f, err := os.Create(filepath.Join(spider.Src, "Spiderfile"))
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+	defer f.Close()
+	if _, err := f.Write(contentByte); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
 	// 添加爬虫到数据库
 	if err := spider.Add(); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)

From d3b4db29ee88daca39dda9b58b540ad721391932 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 22:23:12 +0800
Subject: [PATCH 25/52] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=BA=E5=BA=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/views/spider/SpiderList.vue | 47 +++++++++---------------
 1 file changed, 17 insertions(+), 30 deletions(-)

diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index 41007044..b185574e 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -34,22 +34,6 @@
                :visible.sync="addDialogVisible"
                :before-close="onAddDialogClose">
       <el-tabs :active-name="spiderType">
-        <el-tab-pane name="customized" :label="$t('Customized')">
-          <el-form :model="spiderForm" ref="addCustomizedForm" inline-message>
-            <el-form-item :label="$t('Upload Zip File')" label-width="120px" name="site">
-              <el-upload
-                :action="$request.baseUrl + '/spiders'"
-                :headers="{Authorization:token}"
-                :on-change="onUploadChange"
-                :on-success="onUploadSuccess"
-                :file-list="fileList">
-                <el-button size="small" type="primary" icon="el-icon-upload">{{$t('Upload')}}</el-button>
-              </el-upload>
-            </el-form-item>
-          </el-form>
-          <el-alert type="error" :title="$t('Please zip your spider files from the root directory')"
-                    :closable="false"></el-alert>
-        </el-tab-pane>
         <el-tab-pane name="configurable" :label="$t('Configurable')">
           <el-form :model="spiderForm" ref="addConfigurableForm" inline-message label-width="120px">
             <el-form-item :label="$t('Spider Name')" prop="name" required>
@@ -66,19 +50,23 @@
             <el-button type="primary" @click="onAddConfigurable">{{$t('Add')}}</el-button>
           </div>
         </el-tab-pane>
+        <el-tab-pane name="customized" :label="$t('Customized')">
+          <el-form :model="spiderForm" ref="addCustomizedForm" inline-message>
+            <el-form-item :label="$t('Upload Zip File')" label-width="120px" name="site">
+              <el-upload
+                :action="$request.baseUrl + '/spiders'"
+                :headers="{Authorization:token}"
+                :on-change="onUploadChange"
+                :on-success="onUploadSuccess"
+                :file-list="fileList">
+                <el-button size="small" type="primary" icon="el-icon-upload">{{$t('Upload')}}</el-button>
+              </el-upload>
+            </el-form-item>
+          </el-form>
+          <el-alert type="error" :title="$t('Please zip your spider files from the root directory')"
+                    :closable="false"></el-alert>
+        </el-tab-pane>
       </el-tabs>
-      <!--      <div class="add-spider-wrapper">-->
-      <!--        <div @click="onAddConfigurable">-->
-      <!--          <el-card shadow="hover" class="add-spider-item success">-->
-      <!--            {{$t('Configurable Spider')}}-->
-      <!--          </el-card>-->
-      <!--        </div>-->
-      <!--        <div @click="onAddCustomized">-->
-      <!--          <el-card shadow="hover" class="add-spider-item primary">-->
-      <!--            {{$t('Customized Spider')}}-->
-      <!--          </el-card>-->
-      <!--        </div>-->
-      <!--      </div>-->
     </el-dialog>
     <!--./add dialog-->
 
@@ -323,7 +311,7 @@ export default {
         name: [{ required: true, message: 'Required Field', trigger: 'change' }]
       },
       fileList: [],
-      spiderType: 'customized'
+      spiderType: 'configurable'
     }
   },
   computed: {
@@ -356,7 +344,6 @@ export default {
     onAdd () {
       this.$store.commit('spider/SET_SPIDER_FORM', {})
       this.addDialogVisible = true
-      // this.onAddCustomized()
     },
     onAddConfigurable () {
       this.$refs['addConfigurableForm'].validate(async res => {

From 83e148b31d00166bcfa63e83c8898a2338216819 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 22:37:32 +0800
Subject: [PATCH 26/52] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E7=BF=BB=E8=AF=91?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/i18n/zh.js | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 43ef33ef..a1190253 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -151,6 +151,19 @@ export default {
   'Detail Page Fields': '详情页字段',
   'Detail Page URL': '详情页URL',
   'All': '全部',
+  'Stages': '阶段',
+  'Process': '流程',
+  'Stage Process': '流程图',
+  'Stage Name': '阶段名称',
+  'Start Stage': '开始阶段',
+  'Engine': '引擎',
+  'Selector Type': '选择器类别',
+  'Selector': '选择器',
+  'Is Attribute': '是否为属性',
+  'Next Stage': '下一阶段',
+  'No Next Stage': '没有下一阶段',
+  'Fields': '字段',
+  'Stage': '阶段',
 
   // 爬虫列表
   'Name': '名称',

From 5785e3d890d8efb553392ac071fe8b7edcf64fff Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 28 Nov 2019 22:38:49 +0800
Subject: [PATCH 27/52] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E7=BF=BB=E8=AF=91?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/store/modules/lang.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/store/modules/lang.js b/frontend/src/store/modules/lang.js
index b1e57a04..dc6b8d18 100644
--- a/frontend/src/store/modules/lang.js
+++ b/frontend/src/store/modules/lang.js
@@ -1,5 +1,5 @@
 const state = {
-  lang: window.localStorage.getItem('lang') || 'en'
+  lang: window.localStorage.getItem('lang') || 'zh'
 }
 
 const getters = {

From 975ca35f452740018227591a8e685b4466b15b51 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Fri, 29 Nov 2019 13:00:19 +0800
Subject: [PATCH 28/52] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbug?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 3 +++
 frontend/src/views/spider/SpiderList.vue      | 1 +
 2 files changed, 4 insertions(+)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 014bfd8d..15eba993 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -225,6 +225,9 @@ export default {
 
         // 获取Spiderfile
         this.getSpiderfile()
+
+        // 获取config
+        this.$store.dispatch('spider/getSpiderData', this.spiderForm._id)
       }, 0)
     }
   },
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index b185574e..c756ce52 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -356,6 +356,7 @@ export default {
           this.$message.error(this.$t('Something wrong happened'))
           return
         }
+        await this.$store.dispatch('spider/getSpiderList')
         this.$router.push(`/spiders/${res2.data.data._id}`)
         this.$st.sendEv('爬虫', '添加爬虫-可配置爬虫')
       })

From d7cbaac94fcbce7dff4afabf81609f7b783a27e3 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Fri, 29 Nov 2019 13:00:33 +0800
Subject: [PATCH 29/52] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbug?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/store/modules/file.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/frontend/src/store/modules/file.js b/frontend/src/store/modules/file.js
index 107a82d5..66b84651 100644
--- a/frontend/src/store/modules/file.js
+++ b/frontend/src/store/modules/file.js
@@ -42,8 +42,7 @@ const actions = {
       .then(response => {
         commit('SET_FILE_CONTENT', response.data.data)
       })
-  },
-
+  }
 }
 
 export default {

From 87546f0c88a5c8b581fcbb403bafaa77de215135 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Fri, 29 Nov 2019 13:42:50 +0800
Subject: [PATCH 30/52] =?UTF-8?q?=E5=87=86=E5=A4=87=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E8=87=AA=E5=AE=9A=E4=B9=89=E8=AE=BE?=
 =?UTF-8?q?=E7=BD=AE=E5=8F=98=E9=87=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/entity/config_spider.go               |  27 +-
 backend/routes/config_spider.go               |  10 +-
 backend/services/task.go                      |   6 +
 backend/template/Spiderfile                   |   2 +
 .../template/scrapy/config_spider/settings.py |   8 +-
 frontend/src/components/Config/ConfigList.vue |  16 +
 .../TableView/SettingFieldsTableView.vue      | 349 ++++++++++++++++++
 7 files changed, 402 insertions(+), 16 deletions(-)
 create mode 100644 frontend/src/components/TableView/SettingFieldsTableView.vue

diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go
index 7afb7109..3fe28bc9 100644
--- a/backend/entity/config_spider.go
+++ b/backend/entity/config_spider.go
@@ -1,12 +1,12 @@
 package entity
 
-type Field struct {
-	Name      string `yaml:"name" json:"name"`
-	Css       string `yaml:"css" json:"css"`
-	Xpath     string `yaml:"xpath" json:"xpath"`
-	Attr      string `yaml:"attr" json:"attr"`
-	NextStage string `yaml:"next_stage" json:"next_stage"`
-	Remark    string `yaml:"remark" json:"remark"`
+type ConfigSpiderData struct {
+	Version    string            `yaml:"version" json:"version"`
+	Engine     string            `yaml:"engine" json:"engine"`
+	StartUrl   string            `yaml:"start_url" json:"start_url"`
+	StartStage string            `yaml:"start_stage" json:"start_stage"`
+	Stages     map[string]Stage  `yaml:"stages" json:"stages"`
+	Settings   map[string]string `yaml:"settings" json:"settings"`
 }
 
 type Stage struct {
@@ -20,10 +20,11 @@ type Stage struct {
 	Fields    []Field `yaml:"fields" json:"fields"`
 }
 
-type ConfigSpiderData struct {
-	Version    string           `yaml:"version" json:"version"`
-	Engine     string           `yaml:"engine" json:"engine"`
-	StartUrl   string           `yaml:"start_url" json:"start_url"`
-	StartStage string           `yaml:"start_stage" json:"start_stage"`
-	Stages     map[string]Stage `yaml:"stages" json:"stages"`
+type Field struct {
+	Name      string `yaml:"name" json:"name"`
+	Css       string `yaml:"css" json:"css"`
+	Xpath     string `yaml:"xpath" json:"xpath"`
+	Attr      string `yaml:"attr" json:"attr"`
+	NextStage string `yaml:"next_stage" json:"next_stage"`
+	Remark    string `yaml:"remark" json:"remark"`
 }
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 982af28d..61067f98 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -205,8 +205,8 @@ func PostConfigSpiderSpiderfile(c *gin.Context) {
 		return
 	}
 
-	// 根据序列化后的数据处理爬虫文件
-	if err := services.ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
+	// 校验configData
+	if err := services.ValidateSpiderfile(configData); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return
 	}
@@ -217,6 +217,12 @@ func PostConfigSpiderSpiderfile(c *gin.Context) {
 		return
 	}
 
+	// 根据序列化后的数据处理爬虫文件
+	if err := services.ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
diff --git a/backend/services/task.go b/backend/services/task.go
index 9336e75d..0cd53d86 100644
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -219,12 +219,18 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
 	// 环境变量配置
 	envs := s.Envs
 	if s.Type == constants.Configurable {
+		// 数据库配置
 		envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_HOST", Value: viper.GetString("mongo.host")})
 		envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_PORT", Value: viper.GetString("mongo.port")})
 		envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_DB", Value: viper.GetString("mongo.db")})
 		envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_USERNAME", Value: viper.GetString("mongo.username")})
 		envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_PASSWORD", Value: viper.GetString("mongo.password")})
 		envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_AUTHSOURCE", Value: viper.GetString("mongo.authSource")})
+
+		// 设置配置
+		for envName, envValue := range s.Config.Settings {
+			envs = append(envs, model.Env{Name: "CRAWLAB_SETTING_" + envName, Value: envValue})
+		}
 	}
 	cmd = SetEnv(cmd, envs, t.Id, s.Col)
 
diff --git a/backend/template/Spiderfile b/backend/template/Spiderfile
index 8d0e05cf..d748d5f8 100644
--- a/backend/template/Spiderfile
+++ b/backend/template/Spiderfile
@@ -23,3 +23,5 @@ stages:
     fields:
       - name: "description"
         css: "#product_description + p"
+settings:
+  ROBOTSTXT_OBEY: true
diff --git a/backend/template/scrapy/config_spider/settings.py b/backend/template/scrapy/config_spider/settings.py
index a0112373..3dde4c6c 100644
--- a/backend/template/scrapy/config_spider/settings.py
+++ b/backend/template/scrapy/config_spider/settings.py
@@ -9,7 +9,7 @@
 #     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
 #     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
 
-BOT_NAME = 'config_spider'
+BOT_NAME = 'Crawlab Configurable Spider'
 
 SPIDER_MODULES = ['config_spider.spiders']
 NEWSPIDER_MODULE = 'config_spider.spiders'
@@ -88,3 +88,9 @@ ITEM_PIPELINES = {
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWALAB_SETTING_')]:
+    setting_name = setting_env_name.replace('CRAWLAB_SETTING_', '')
+    setting_value = os.environ.get('setting_env_name')
+    locals()[setting_name] = setting_value
+
diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 15eba993..ac66e8d4 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -182,6 +182,20 @@
       </el-tab-pane>
       <!--./Graph-->
 
+      <!--Setting-->
+      <el-tab-pane name="setting" :label="$t('Setting')">
+        <setting-fields-table-view
+          type="list"
+          :fields="spiderForm.settings"
+        />
+        <fields-table-view
+          type="list"
+          title="List Page Fields"
+          :fields="stage.fields"
+        />
+      </el-tab-pane>
+      <!--./Setting-->
+
       <!--Spiderfile-->
       <el-tab-pane name="spiderfile" label="Spiderfile">
         <div class="spiderfile-actions">
@@ -209,10 +223,12 @@ import CrawlConfirmDialog from '../Common/CrawlConfirmDialog'
 import 'codemirror/lib/codemirror.js'
 import 'codemirror/mode/yaml/yaml.js'
 import FileDetail from '../File/FileDetail'
+import SettingFieldsTableView from '../TableView/SettingFieldsTableView'
 
 export default {
   name: 'ConfigList',
   components: {
+    SettingFieldsTableView,
     FileDetail,
     CrawlConfirmDialog,
     FieldsTableView
diff --git a/frontend/src/components/TableView/SettingFieldsTableView.vue b/frontend/src/components/TableView/SettingFieldsTableView.vue
new file mode 100644
index 00000000..cfba6d20
--- /dev/null
+++ b/frontend/src/components/TableView/SettingFieldsTableView.vue
@@ -0,0 +1,349 @@
+<template>
+  <div class="setting-fields-table-view">
+    <!--    <el-row class="button-group-container">-->
+    <!--      <label class="title">{{$t(this.title)}}</label>-->
+    <!--      <div class="button-group">-->
+    <!--        <el-button type="primary" size="small" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>-->
+    <!--      </div>-->
+    <!--    </el-row>-->
+    <el-row>
+      <el-table :data="fields"
+                class="table edit"
+                :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
+                :cell-style="getCellClassStyle"
+      >
+        <el-table-column class-name="action" width="80px" align="right">
+          <template slot-scope="scope">
+            <i class="action-item el-icon-copy-document" @click="onCopyField(scope.row)"></i>
+            <i class="action-item el-icon-remove-outline" @click="onRemoveField(scope.row)"></i>
+            <i class="action-item el-icon-circle-plus-outline" @click="onAddField(scope.row)"></i>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Field Name')" width="150px">
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.name"
+                      :placeholder="$t('Field Name')"
+                      @change="onNameChange(scope.row)"
+            />
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Selector Type')" width="150px" align="center" class-name="selector-type">
+          <template slot-scope="scope">
+            <span class="button-selector-item" @click="onClickSelectorType(scope.row, 'css')">
+              <el-tag
+                :class="scope.row.css ? 'active' : 'inactive'"
+                type="success"
+              >
+                CSS
+              </el-tag>
+            </span>
+            <span class="button-selector-item" @click="onClickSelectorType(scope.row, 'xpath')">
+              <el-tag
+                :class="scope.row.xpath ? 'active' : 'inactive'"
+                type="primary"
+              >
+                XPath
+              </el-tag>
+            </span>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Selector')" width="200px">
+          <template slot-scope="scope">
+            <template v-if="scope.row.css">
+              <el-input v-model="scope.row.css" :placeholder="$t('CSS / XPath')"></el-input>
+            </template>
+            <template v-else>
+              <el-input v-model="scope.row.xpath" :placeholder="$t('CSS / XPath')"></el-input>
+            </template>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Is Attribute')" width="150px" align="center">
+          <template slot-scope="scope">
+            <span class="button-selector-item" @click="onClickIsAttribute(scope.row, false)">
+              <el-tag
+                :class="!scope.row.attr ? 'active' : 'inactive'"
+                type="success"
+              >
+                {{$t('Text')}}
+              </el-tag>
+            </span>
+            <span class="button-selector-item" @click="onClickIsAttribute(scope.row, true)">
+              <el-tag
+                :class="scope.row.attr ? 'active' : 'inactive'"
+                type="primary"
+              >
+                {{$t('Attribute')}}
+              </el-tag>
+            </span>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Attribute')" width="200px">
+          <template slot-scope="scope">
+            <template v-if="scope.row.attr">
+              <el-input v-model="scope.row.attr" :placeholder="$t('Attribute')"/>
+            </template>
+            <template v-else>
+              <span style="margin-left: 15px; color: lightgrey">
+                N/A
+              </span>
+            </template>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Next Stage')" width="250px">
+          <template slot-scope="scope">
+            <el-select
+              v-model="scope.row.next_stage"
+              :class="!scope.row.next_stage ? 'disabled' : ''"
+              @change="onChangeNextStage(scope.row)"
+            >
+              <el-option :label="$t('No Next Stage')" value=""/>
+              <el-option v-for="n in stageNames" :key="n" :label="n" :value="n"/>
+            </el-select>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Remark')" width="auto" min-width="120px">
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.remark" :placeholder="$t('Remark')"/>
+          </template>
+        </el-table-column>
+      </el-table>
+    </el-row>
+  </div>
+</template>
+
+<script>
+import {
+  mapState
+} from 'vuex'
+
+export default {
+  name: 'SettingFieldsTableView',
+  props: {
+    type: {
+      type: String,
+      default: 'list'
+    },
+    title: {
+      type: String,
+      default: ''
+    },
+    stageNames: {
+      type: Array,
+      default () {
+        return []
+      }
+    },
+    fields: {
+      type: Array,
+      default () {
+        return []
+      }
+    }
+  },
+  computed: {
+    ...mapState('spider', [
+      'spiderForm'
+    ])
+  },
+  methods: {
+    addField () {
+      this.fields.push({
+        type: 'css',
+        extract_type: 'text'
+      })
+      this.$st.sendEv('爬虫详情-配置', '添加字段')
+    },
+    deleteField (index) {
+      this.fields.splice(index, 1)
+      this.$st.sendEv('爬虫详情-配置', '删除字段')
+    },
+    onNameChange (row) {
+      if (this.fields.filter(d => d.name === row.name).length > 1) {
+        this.$message.error(this.$t(`Duplicated field names for ${row.name}`))
+      }
+      this.$st.sendEv('爬虫详情-配置', '更改字段')
+    },
+    onCheck (row) {
+      this.fields.forEach(d => {
+        if (row.name !== d.name) {
+          this.$set(d, 'is_detail', false)
+        }
+      })
+      this.$st.sendEv('爬虫详情-配置', '设置详情页URL')
+    },
+    onClickSelectorType (row, selectorType) {
+      if (selectorType === 'css') {
+        if (row.xpath) this.$set(row, 'xpath', '')
+        if (!row.css) this.$set(row, 'css', 'body')
+      } else {
+        if (row.css) this.$set(row, 'css', '')
+        if (!row.xpath) this.$set(row, 'xpath', '//body')
+      }
+    },
+    onClickIsAttribute (row, isAttribute) {
+      if (!isAttribute) {
+        // 文本
+        if (row.attr) this.$set(row, 'attr', '')
+      } else {
+        // 属性
+        if (!row.attr) this.$set(row, 'attr', 'href')
+      }
+    },
+    onCopyField (row) {
+      for (let i = 0; i < this.fields.length; i++) {
+        if (row.name === this.fields[i].name) {
+          this.fields.splice(i, 0, JSON.parse(JSON.stringify(row)))
+          break
+        }
+      }
+    },
+    onRemoveField (row) {
+      for (let i = 0; i < this.fields.length; i++) {
+        if (row.name === this.fields[i].name) {
+          this.fields.splice(i, 1)
+          break
+        }
+      }
+      if (this.fields.length === 0) {
+        this.fields.push({
+          css: 'body',
+          next_stage: ''
+        })
+      }
+    },
+    onAddField (row) {
+      for (let i = 0; i < this.fields.length; i++) {
+        if (row.name === this.fields[i].name) {
+          this.fields.splice(i + 1, 0, {
+            name: `field_${Math.floor(new Date().getTime()).toString()}`,
+            css: 'body',
+            next_stage: ''
+          })
+          break
+        }
+      }
+    },
+    getCellClassStyle ({ row, columnIndex }) {
+      if (columnIndex === 1) {
+        // 字段名称
+        if (!row.name) {
+          return {
+            'border': '1px solid red'
+          }
+        }
+      } else if (columnIndex === 3) {
+        // 选择器
+        if (!row.css && !row.xpath) {
+          return {
+            'border': '1px solid red'
+          }
+        }
+      }
+    },
+    onChangeNextStage (row) {
+      this.fields.forEach(f => {
+        if (f.name !== row.name) {
+          this.$set(f, 'next_stage', '')
+        }
+      })
+    }
+  }
+}
+</script>
+
+<style scoped>
+  .el-table.edit >>> .el-table__body td {
+    padding: 0;
+  }
+
+  .el-table.edit >>> .el-table__body td .cell {
+    padding: 0;
+    font-size: 12px;
+  }
+
+  .el-table.edit >>> .el-input__inner:hover {
+    text-decoration: underline;
+  }
+
+  .el-table.edit >>> .el-input__inner {
+    height: 36px;
+    border: none;
+    border-radius: 0;
+    font-size: 12px;
+  }
+
+  .el-table.edit >>> .el-select .el-input .el-select__caret {
+    line-height: 36px;
+  }
+
+  .el-table.edit >>> .button-selector-item {
+    cursor: pointer;
+    margin: 0 5px;
+  }
+
+  .el-table.edit >>> .el-tag.inactive {
+    opacity: 0.5;
+  }
+
+  .el-table.edit >>> .action {
+    background: none !important;
+    border: none;
+  }
+
+  .el-table.edit >>> tr {
+    border: none;
+  }
+
+  .el-table.edit >>> tr th {
+    border-right: 1px solid rgb(220, 223, 230);
+  }
+
+  .el-table.edit >>> tr td:nth-child(2) {
+    border-left: 1px solid rgb(220, 223, 230);
+  }
+
+  .el-table.edit >>> tr td {
+    border-right: 1px solid rgb(220, 223, 230);
+  }
+
+  .el-table.edit::before {
+    background: none;
+  }
+
+  .el-table.edit >>> .action-item {
+    font-size: 14px;
+    margin-right: 5px;
+    cursor: pointer;
+  }
+
+  .el-table.edit >>> .action-item:last-child {
+    margin-right: 10px;
+  }
+
+  .button-group-container {
+    /*display: inline-block;*/
+    /*width: 100%;*/
+  }
+
+  .button-group-container .title {
+    float: left;
+    line-height: 32px;
+  }
+
+  .button-group-container .button-group {
+    float: right;
+  }
+
+  .action-button-group {
+    display: flex;
+    margin-left: 10px;
+  }
+
+  .action-button-group >>> .el-checkbox__label {
+    font-size: 12px;
+  }
+
+  .el-table.edit >>> .el-select.disabled .el-input__inner {
+    color: lightgrey;
+  }
+</style>

From 32b162e00ca7cbf406318fa872d3f2df2bf68e29 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Fri, 29 Nov 2019 13:46:41 +0800
Subject: [PATCH 31/52] =?UTF-8?q?=E5=87=86=E5=A4=87=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E8=AE=BE=E7=BD=AE=E9=85=8D=E7=BD=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index ac66e8d4..bd909cf0 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -184,15 +184,10 @@
 
       <!--Setting-->
       <el-tab-pane name="setting" :label="$t('Setting')">
-        <setting-fields-table-view
-          type="list"
-          :fields="spiderForm.settings"
-        />
-        <fields-table-view
-          type="list"
-          title="List Page Fields"
-          :fields="stage.fields"
-        />
+<!--        <setting-fields-table-view-->
+<!--          type="list"-->
+<!--          :fields="spiderForm.settings"-->
+<!--        />-->
       </el-tab-pane>
       <!--./Setting-->
 

From ff6ab9a28107f7f70b219f842d2b7a9eae009b52 Mon Sep 17 00:00:00 2001
From: hantmac <hantmac@outlook.com>
Date: Fri, 29 Nov 2019 13:52:03 +0800
Subject: [PATCH 32/52] =?UTF-8?q?bug=20fix:=E4=BF=AE=E5=A4=8D=E6=89=8B?=
 =?UTF-8?q?=E5=8A=A8=E6=88=96=E8=80=85=E8=87=AA=E5=8A=A8=E5=88=A0=E9=99=A4?=
 =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=90=8E=E7=9A=84=E6=8A=A5=E9=94=99?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/services/task.go | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/backend/services/task.go b/backend/services/task.go
index 9336e75d..14a609ad 100644
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -468,6 +468,29 @@ func GetTaskLog(id string) (logStr string, err error) {
 	}
 
 	if IsMasterNode(task.NodeId.Hex()) {
+		if !utils.Exists(task.LogPath) {
+			fileDir, err := MakeLogDir(task)
+
+			if err != nil {
+				log.Errorf(err.Error())
+			}
+
+			fileP := GetLogFilePaths(fileDir)
+
+			// 获取日志文件路径
+			fLog, err := os.Create(fileP)
+			defer fLog.Close()
+			if err != nil {
+				log.Errorf("create task log file error: %s", fileP)
+				debug.PrintStack()
+			}
+			task.LogPath = fileP
+			if err := task.Save(); err != nil {
+				log.Errorf(err.Error())
+				debug.PrintStack()
+			}
+
+		}
 		// 若为主节点，获取本机日志
 		logBytes, err := model.GetLocalLog(task.LogPath)
 		if err != nil {

From 5405dba5406cd67d56218185cd6bfcbe96c9ffb8 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Sat, 30 Nov 2019 10:58:54 +0800
Subject: [PATCH 33/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E9=98=B6=E6=AE=B5=E8=AE=BE=E7=BD=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/services/spider.go                    |  15 +-
 frontend/src/components/Config/ConfigList.vue | 155 ++++++++++++++++--
 frontend/src/i18n/zh.js                       |   2 +
 3 files changed, 152 insertions(+), 20 deletions(-)

diff --git a/backend/services/spider.go b/backend/services/spider.go
index aa97b4ad..3922d822 100644
--- a/backend/services/spider.go
+++ b/backend/services/spider.go
@@ -116,12 +116,15 @@ func PublishAllSpiders() {
 
 // 发布爬虫
 func PublishSpider(spider model.Spider) {
-	// 查询gf file，不存在则标记为爬虫文件不存在
-	gfFile := model.GetGridFs(spider.FileId)
-	if gfFile == nil {
-		spider.FileId = constants.ObjectIdNull
-		_ = spider.Save()
-		return
+	var gfFile *model.GridFs
+	if spider.FileId.Hex() != constants.ObjectIdNull {
+		// 查询gf file，不存在则标记为爬虫文件不存在
+		gfFile = model.GetGridFs(spider.FileId)
+		if gfFile == nil {
+			spider.FileId = constants.ObjectIdNull
+			_ = spider.Save()
+			return
+		}
 	}
 
 	// 如果FileId为空，表示还没有上传爬虫到GridFS，则跳过
diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index bd909cf0..d08b1109 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -139,18 +139,22 @@
           >
             <template slot="title">
               <ul class="stage-list">
-                <li class="stage-item" style="flex-basis: 80px; justify-content: flex-end"
+                <!--actions-->
+                <li class="stage-item" style="min-width: 80px; flex-basis: 80px; justify-content: flex-end"
                     @click="$event.stopPropagation()">
                   <i class="action-item el-icon-copy-document" @click="onCopyStage(stage)"></i>
                   <i class="action-item el-icon-remove-outline" @click="onRemoveStage(stage)"></i>
                   <i class="action-item el-icon-circle-plus-outline" @click="onAddStage(stage)"></i>
                 </li>
-                <li class="stage-item" @click="$event.stopPropagation()">
+                <!--./actions-->
+
+                <!--stage name-->
+                <li class="stage-item" style="min-width: 240px" @click="$event.stopPropagation()">
                   <label>{{$t('Stage Name')}}: </label>
                   <div v-if="!stage.isEdit" @click="onEditStage(stage)" class="text-wrapper">
-                <span class="text">
-                  {{stage.name}}
-                </span>
+                    <span class="text">
+                      {{stage.name}}
+                    </span>
                     <i class="el-icon-edit-outline"></i>
                   </div>
                   <el-input
@@ -163,6 +167,72 @@
                     @blur="stage.isEdit = false"
                   />
                 </li>
+                <!--./stage name-->
+
+                <!--list-->
+                <li class="stage-item" style="min-width: 240px">
+                  <label>{{$t('Is List')}}: </label>
+                  <el-checkbox
+                    style="text-align: left; flex-basis: 20px; margin-right: 5px"
+                    :value="isList(stage)"
+                    @change="onCheckIsList($event, stage)"
+                  />
+                  <el-popover v-if="isList(stage)" placement="top">
+                    <div>
+                      <el-tag :class="stage.list_css ? 'active' : 'inactive'" type="success"
+                              @click="onSelectStageListType(stage, 'css')">CSS
+                      </el-tag>
+                      <el-tag :class="!stage.list_css ? 'active' : 'inactive'" type="primary"
+                              @click="onSelectStageListType(stage, 'xpath')">XPath
+                      </el-tag>
+                    </div>
+                    <div class="list-selector" style="margin-top: 5px; width: 240px">
+                      <el-input v-if="stage.list_css" v-model="stage.list_css"/>
+                      <el-input v-else v-model="stage.list_xpath"/>
+                    </div>
+                    <el-tag v-if="stage.list_css" type="success" slot="reference"
+                            @click="onClickStageList($event, stage, 'css')">CSS
+                    </el-tag>
+                    <el-tag v-else type="primary" slot="reference"
+                            @click="onClickStageList($event, stage, 'xpath')">
+                      XPath
+                    </el-tag>
+                  </el-popover>
+                </li>
+                <!--./list-->
+
+                <!--pagination-->
+                <li class="stage-item" style="min-width: 240px">
+                  <label>{{$t('Pagination')}}: </label>
+                  <el-checkbox
+                    style="text-align: left; flex-basis: 20px; margin-right: 5px"
+                    :value="isPage(stage)"
+                    @change="onCheckIsPage($event, stage)"
+                  />
+                  <el-popover v-if="isPage(stage)" placement="top">
+                    <div>
+                      <el-tag :class="stage.page_css ? 'active' : 'inactive'" type="success"
+                              @click="onSelectStagePageType(stage, 'css')">CSS
+                      </el-tag>
+                      <el-tag :class="!stage.page_css ? 'active' : 'inactive'" type="primary"
+                              @click="onSelectStagePageType(stage, 'xpath')">XPath
+                      </el-tag>
+                    </div>
+                    <div class="page-selector" style="margin-top: 5px; width: 240px">
+                      <el-input v-if="stage.page_css" v-model="stage.page_css"/>
+                      <el-input v-else v-model="stage.page_xpath"/>
+                    </div>
+                    <el-tag v-if="stage.page_css" type="success" slot="reference"
+                            @click="onClickStagePage($event, stage, 'css')">CSS
+                    </el-tag>
+                    <el-tag v-else type="primary" slot="reference"
+                            @click="onClickStagePage($event, stage, 'xpath')">
+                      XPath
+                    </el-tag>
+                  </el-popover>
+                </li>
+                <!--./pagination-->
+
               </ul>
             </template>
             <fields-table-view
@@ -183,11 +253,10 @@
       <!--./Graph-->
 
       <!--Setting-->
-      <el-tab-pane name="setting" :label="$t('Setting')">
-<!--        <setting-fields-table-view-->
-<!--          type="list"-->
-<!--          :fields="spiderForm.settings"-->
-<!--        />-->
+      <el-tab-pane name="settings" :label="$t('Settings')">
+        <setting-fields-table-view
+          type="list"
+        />
       </el-tab-pane>
       <!--./Setting-->
 
@@ -600,6 +669,56 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
       } catch (e) {
         this.$message.error('Something wrong happened')
       }
+    },
+    isList (stage) {
+      return !!stage.list_css || !!stage.list_xpath
+    },
+    onCheckIsList (value, stage) {
+      if (value) {
+        if (!stage.list_css && !stage.list_xpath) {
+          stage.list_css = 'body'
+        }
+      } else {
+        stage.list_css = ''
+        stage.list_xpath = ''
+      }
+    },
+    onClickStageList ($event, stage, type) {
+      $event.stopPropagation()
+    },
+    onSelectStageListType (stage, type) {
+      if (type === 'css') {
+        if (!stage.list_css) stage.list_css = 'body'
+        stage.list_xpath = ''
+      } else {
+        if (!stage.list_xpath) stage.list_xpath = '//body'
+        stage.list_css = ''
+      }
+    },
+    isPage (stage) {
+      return !!stage.page_css || !!stage.page_xpath
+    },
+    onCheckIsPage (value, stage) {
+      if (value) {
+        if (!stage.page_css && !stage.page_xpath) {
+          stage.page_css = 'body'
+        }
+      } else {
+        stage.page_css = ''
+        stage.page_xpath = ''
+      }
+    },
+    onClickStagePage ($event, stage, type) {
+      $event.stopPropagation()
+    },
+    onSelectStagePageType (stage, type) {
+      if (type === 'css') {
+        if (!stage.page_css) stage.page_css = 'body'
+        stage.page_xpath = ''
+      } else {
+        if (!stage.page_xpath) stage.page_xpath = '//body'
+        stage.page_css = ''
+      }
     }
   },
   mounted () {
@@ -653,12 +772,19 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
     font-weight: bolder;
   }
 
-  .selector-type-item > .el-tag.inactive {
+  .el-tag {
+    margin-right: 5px;
+    font-weight: bolder;
+    cursor: pointer;
+  }
+
+  .el-tag.inactive {
     opacity: 0.5;
   }
 
   .stage-list {
-    width: calc(80px + 320px);
+    width: 100%;
+    /*width: calc(80px + 320px);*/
     display: flex;
     flex-wrap: wrap;
     list-style: none;
@@ -667,8 +793,8 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
   }
 
   .stage-list .stage-item {
-    flex-basis: 320px;
-    width: 320px;
+    /*flex-basis: 320px;*/
+    min-width: 120px;
     display: flex;
     align-items: center;
   }
@@ -677,6 +803,7 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
     flex-basis: 90px;
     margin-right: 10px;
     justify-self: flex-end;
+    text-align: right;
   }
 
   .stage-list .stage-item .el-input {
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index a1190253..4d2883e1 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -164,6 +164,8 @@ export default {
   'No Next Stage': '没有下一阶段',
   'Fields': '字段',
   'Stage': '阶段',
+  'Is List': '是否为列表',
+  'Pagination': '分页',
 
   // 爬虫列表
   'Name': '名称',

From bd0ccdcb38ddfabb7b5e4b6d0b71075ec1550836 Mon Sep 17 00:00:00 2001
From: Bo-Yi Wu <appleboy.tw@gmail.com>
Date: Sun, 1 Dec 2019 18:16:39 +0800
Subject: [PATCH 34/52] chore: fix fmt warning.

---
 backend/entity/common.go  | 10 +++++-----
 backend/mock/node_test.go | 12 ++++++------
 backend/mock/stats.go     |  2 --
 backend/mock/system.go    |  2 +-
 backend/mock/user.go      |  2 +-
 5 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/backend/entity/common.go b/backend/entity/common.go
index 332cc494..c46ae4f9 100644
--- a/backend/entity/common.go
+++ b/backend/entity/common.go
@@ -3,15 +3,15 @@ package entity
 import "strconv"
 
 type Page struct {
-	Skip int
-	Limit int
-	PageNum int
+	Skip     int
+	Limit    int
+	PageNum  int
 	PageSize int
 }
 
-func (p *Page)GetPage(pageNum string, pageSize string) {
+func (p *Page) GetPage(pageNum string, pageSize string) {
 	p.PageNum, _ = strconv.Atoi(pageNum)
 	p.PageSize, _ = strconv.Atoi(pageSize)
 	p.Skip = p.PageSize * (p.PageNum - 1)
 	p.Limit = p.PageSize
-}
\ No newline at end of file
+}
diff --git a/backend/mock/node_test.go b/backend/mock/node_test.go
index 669cafc5..abd568c2 100644
--- a/backend/mock/node_test.go
+++ b/backend/mock/node_test.go
@@ -42,12 +42,12 @@ func init() {
 	app.DELETE("/tasks/:id", DeleteTask)                           // 删除任务
 	app.GET("/tasks/:id/results", GetTaskResults)                  // 任务结果
 	app.GET("/tasks/:id/results/download", DownloadTaskResultsCsv) // 下载任务结果
-	app.GET("/spiders", GetSpiderList)              // 爬虫列表
-	app.GET("/spiders/:id", GetSpider)              // 爬虫详情
-	app.POST("/spiders/:id", PostSpider)            // 修改爬虫
-	app.DELETE("/spiders/:id",DeleteSpider)        // 删除爬虫
-	app.GET("/spiders/:id/tasks",GetSpiderTasks)   // 爬虫任务列表
-	app.GET("/spiders/:id/dir",GetSpiderDir)       // 爬虫目录
+	app.GET("/spiders", GetSpiderList)                             // 爬虫列表
+	app.GET("/spiders/:id", GetSpider)                             // 爬虫详情
+	app.POST("/spiders/:id", PostSpider)                           // 修改爬虫
+	app.DELETE("/spiders/:id", DeleteSpider)                       // 删除爬虫
+	app.GET("/spiders/:id/tasks", GetSpiderTasks)                  // 爬虫任务列表
+	app.GET("/spiders/:id/dir", GetSpiderDir)                      // 爬虫目录
 }
 
 //mock test, test data in ./mock
diff --git a/backend/mock/stats.go b/backend/mock/stats.go
index db2348c6..f0227da9 100644
--- a/backend/mock/stats.go
+++ b/backend/mock/stats.go
@@ -6,8 +6,6 @@ import (
 	"net/http"
 )
 
-
-
 var taskDailyItems = []model.TaskDailyItem{
 	{
 		Date:               "2019/08/19",
diff --git a/backend/mock/system.go b/backend/mock/system.go
index c4807247..f33e02ba 100644
--- a/backend/mock/system.go
+++ b/backend/mock/system.go
@@ -1 +1 @@
-package mock
\ No newline at end of file
+package mock
diff --git a/backend/mock/user.go b/backend/mock/user.go
index c4807247..f33e02ba 100644
--- a/backend/mock/user.go
+++ b/backend/mock/user.go
@@ -1 +1 @@
-package mock
\ No newline at end of file
+package mock

From 0eba6754ae6fe81d50eff244900515b564601341 Mon Sep 17 00:00:00 2001
From: hantmac <hantmac@outlook.com>
Date: Mon, 2 Dec 2019 10:11:57 +0800
Subject: [PATCH 35/52] fix panic because of interface{} is nil

---
 backend/services/task.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/backend/services/task.go b/backend/services/task.go
index 14a609ad..e862c36e 100644
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -302,9 +302,11 @@ func SaveTaskResultCount(id string) func() {
 
 // 执行任务
 func ExecuteTask(id int) {
-	if flag, _ := LockList.Load(id); flag.(bool) {
-		log.Debugf(GetWorkerPrefix(id) + "正在执行任务...")
-		return
+	if flag, ok := LockList.Load(id); ok {
+		if flag.(bool) {
+			log.Debugf(GetWorkerPrefix(id) + "正在执行任务...")
+		}
+
 	}
 
 	// 上锁

From f58c42c6be467066b1ec27e35a0fceca15913420 Mon Sep 17 00:00:00 2001
From: hantmac <hantmac@outlook.com>
Date: Mon, 2 Dec 2019 11:14:15 +0800
Subject: [PATCH 36/52] fix

---
 backend/services/task.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/services/task.go b/backend/services/task.go
index e862c36e..25b15aeb 100644
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -305,6 +305,7 @@ func ExecuteTask(id int) {
 	if flag, ok := LockList.Load(id); ok {
 		if flag.(bool) {
 			log.Debugf(GetWorkerPrefix(id) + "正在执行任务...")
+			return
 		}
 
 	}

From 7c19aa1d757d994312b97e159dd8df737f5dd362 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Mon, 2 Dec 2019 12:43:44 +0800
Subject: [PATCH 37/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E8=AE=BE=E7=BD=AE=E9=A1=B5=E9=9D=A2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue |   5 +
 .../components/TableView/FieldsTableView.vue  |  15 ++-
 .../TableView/SettingFieldsTableView.vue      | 116 ++++--------------
 frontend/src/i18n/zh.js                       |   1 +
 4 files changed, 38 insertions(+), 99 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index d08b1109..2c730b3b 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -254,6 +254,11 @@
 
       <!--Setting-->
       <el-tab-pane name="settings" :label="$t('Settings')">
+        <div class="actions" style="text-align: right;margin-bottom: 10px">
+          <el-button type="success" size="small">
+            {{$t('Save')}}
+          </el-button>
+        </div>
         <setting-fields-table-view
           type="list"
         />
diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue
index c9f25c89..40c276de 100644
--- a/frontend/src/components/TableView/FieldsTableView.vue
+++ b/frontend/src/components/TableView/FieldsTableView.vue
@@ -61,7 +61,7 @@
           <template slot-scope="scope">
             <span class="button-selector-item" @click="onClickIsAttribute(scope.row, false)">
               <el-tag
-                :class="!scope.row.attr ? 'active' : 'inactive'"
+                :class="!isShowAttr(scope.row) ? 'active' : 'inactive'"
                 type="success"
               >
                 {{$t('Text')}}
@@ -69,7 +69,7 @@
             </span>
             <span class="button-selector-item" @click="onClickIsAttribute(scope.row, true)">
               <el-tag
-                :class="scope.row.attr ? 'active' : 'inactive'"
+                :class="isShowAttr(scope.row) ? 'active' : 'inactive'"
                 type="primary"
               >
                 {{$t('Attribute')}}
@@ -79,8 +79,8 @@
         </el-table-column>
         <el-table-column :label="$t('Attribute')" width="200px">
           <template slot-scope="scope">
-            <template v-if="scope.row.attr">
-              <el-input v-model="scope.row.attr" :placeholder="$t('Attribute')"/>
+            <template v-if="isShowAttr(scope.row)">
+              <el-input v-model="scope.row.attr" :placeholder="$t('Attribute')" @change="onAttrChange(scope.row)"/>
             </template>
             <template v-else>
               <span style="margin-left: 15px; color: lightgrey">
@@ -188,6 +188,7 @@ export default {
         // 属性
         if (!row.attr) this.$set(row, 'attr', 'href')
       }
+      this.$set(row, 'isAttrChange', false)
     },
     onCopyField (row) {
       for (let i = 0; i < this.fields.length; i++) {
@@ -246,6 +247,12 @@ export default {
           this.$set(f, 'next_stage', '')
         }
       })
+    },
+    onAttrChange (row) {
+      this.$set(row, 'isAttrChange', !row.attr)
+    },
+    isShowAttr (row) {
+      return (row.attr || row.isAttrChange)
     }
   }
 }
diff --git a/frontend/src/components/TableView/SettingFieldsTableView.vue b/frontend/src/components/TableView/SettingFieldsTableView.vue
index cfba6d20..77caf85c 100644
--- a/frontend/src/components/TableView/SettingFieldsTableView.vue
+++ b/frontend/src/components/TableView/SettingFieldsTableView.vue
@@ -1,5 +1,5 @@
 <template>
-  <div class="setting-fields-table-view">
+  <div class="setting-list-table-view">
     <!--    <el-row class="button-group-container">-->
     <!--      <label class="title">{{$t(this.title)}}</label>-->
     <!--      <div class="button-group">-->
@@ -7,7 +7,7 @@
     <!--      </div>-->
     <!--    </el-row>-->
     <el-row>
-      <el-table :data="fields"
+      <el-table :data="list"
                 class="table edit"
                 :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
                 :cell-style="getCellClassStyle"
@@ -19,88 +19,14 @@
             <i class="action-item el-icon-circle-plus-outline" @click="onAddField(scope.row)"></i>
           </template>
         </el-table-column>
-        <el-table-column :label="$t('Field Name')" width="150px">
+        <el-table-column :label="$t('Name')" width="150px">
           <template slot-scope="scope">
             <el-input v-model="scope.row.name"
-                      :placeholder="$t('Field Name')"
+                      :placeholder="$t('Name')"
                       @change="onNameChange(scope.row)"
             />
           </template>
         </el-table-column>
-        <el-table-column :label="$t('Selector Type')" width="150px" align="center" class-name="selector-type">
-          <template slot-scope="scope">
-            <span class="button-selector-item" @click="onClickSelectorType(scope.row, 'css')">
-              <el-tag
-                :class="scope.row.css ? 'active' : 'inactive'"
-                type="success"
-              >
-                CSS
-              </el-tag>
-            </span>
-            <span class="button-selector-item" @click="onClickSelectorType(scope.row, 'xpath')">
-              <el-tag
-                :class="scope.row.xpath ? 'active' : 'inactive'"
-                type="primary"
-              >
-                XPath
-              </el-tag>
-            </span>
-          </template>
-        </el-table-column>
-        <el-table-column :label="$t('Selector')" width="200px">
-          <template slot-scope="scope">
-            <template v-if="scope.row.css">
-              <el-input v-model="scope.row.css" :placeholder="$t('CSS / XPath')"></el-input>
-            </template>
-            <template v-else>
-              <el-input v-model="scope.row.xpath" :placeholder="$t('CSS / XPath')"></el-input>
-            </template>
-          </template>
-        </el-table-column>
-        <el-table-column :label="$t('Is Attribute')" width="150px" align="center">
-          <template slot-scope="scope">
-            <span class="button-selector-item" @click="onClickIsAttribute(scope.row, false)">
-              <el-tag
-                :class="!scope.row.attr ? 'active' : 'inactive'"
-                type="success"
-              >
-                {{$t('Text')}}
-              </el-tag>
-            </span>
-            <span class="button-selector-item" @click="onClickIsAttribute(scope.row, true)">
-              <el-tag
-                :class="scope.row.attr ? 'active' : 'inactive'"
-                type="primary"
-              >
-                {{$t('Attribute')}}
-              </el-tag>
-            </span>
-          </template>
-        </el-table-column>
-        <el-table-column :label="$t('Attribute')" width="200px">
-          <template slot-scope="scope">
-            <template v-if="scope.row.attr">
-              <el-input v-model="scope.row.attr" :placeholder="$t('Attribute')"/>
-            </template>
-            <template v-else>
-              <span style="margin-left: 15px; color: lightgrey">
-                N/A
-              </span>
-            </template>
-          </template>
-        </el-table-column>
-        <el-table-column :label="$t('Next Stage')" width="250px">
-          <template slot-scope="scope">
-            <el-select
-              v-model="scope.row.next_stage"
-              :class="!scope.row.next_stage ? 'disabled' : ''"
-              @change="onChangeNextStage(scope.row)"
-            >
-              <el-option :label="$t('No Next Stage')" value=""/>
-              <el-option v-for="n in stageNames" :key="n" :label="n" :value="n"/>
-            </el-select>
-          </template>
-        </el-table-column>
         <el-table-column :label="$t('Remark')" width="auto" min-width="120px">
           <template slot-scope="scope">
             <el-input v-model="scope.row.remark" :placeholder="$t('Remark')"/>
@@ -133,7 +59,7 @@ export default {
         return []
       }
     },
-    fields: {
+    list: {
       type: Array,
       default () {
         return []
@@ -147,24 +73,24 @@ export default {
   },
   methods: {
     addField () {
-      this.fields.push({
+      this.list.push({
         type: 'css',
         extract_type: 'text'
       })
       this.$st.sendEv('爬虫详情-配置', '添加字段')
     },
     deleteField (index) {
-      this.fields.splice(index, 1)
+      this.list.splice(index, 1)
       this.$st.sendEv('爬虫详情-配置', '删除字段')
     },
     onNameChange (row) {
-      if (this.fields.filter(d => d.name === row.name).length > 1) {
+      if (this.list.filter(d => d.name === row.name).length > 1) {
         this.$message.error(this.$t(`Duplicated field names for ${row.name}`))
       }
       this.$st.sendEv('爬虫详情-配置', '更改字段')
     },
     onCheck (row) {
-      this.fields.forEach(d => {
+      this.list.forEach(d => {
         if (row.name !== d.name) {
           this.$set(d, 'is_detail', false)
         }
@@ -190,31 +116,31 @@ export default {
       }
     },
     onCopyField (row) {
-      for (let i = 0; i < this.fields.length; i++) {
-        if (row.name === this.fields[i].name) {
-          this.fields.splice(i, 0, JSON.parse(JSON.stringify(row)))
+      for (let i = 0; i < this.list.length; i++) {
+        if (row.name === this.list[i].name) {
+          this.list.splice(i, 0, JSON.parse(JSON.stringify(row)))
           break
         }
       }
     },
     onRemoveField (row) {
-      for (let i = 0; i < this.fields.length; i++) {
-        if (row.name === this.fields[i].name) {
-          this.fields.splice(i, 1)
+      for (let i = 0; i < this.list.length; i++) {
+        if (row.name === this.list[i].name) {
+          this.list.splice(i, 1)
           break
         }
       }
-      if (this.fields.length === 0) {
-        this.fields.push({
+      if (this.list.length === 0) {
+        this.list.push({
           css: 'body',
           next_stage: ''
         })
       }
     },
     onAddField (row) {
-      for (let i = 0; i < this.fields.length; i++) {
-        if (row.name === this.fields[i].name) {
-          this.fields.splice(i + 1, 0, {
+      for (let i = 0; i < this.list.length; i++) {
+        if (row.name === this.list[i].name) {
+          this.list.splice(i + 1, 0, {
             name: `field_${Math.floor(new Date().getTime()).toString()}`,
             css: 'body',
             next_stage: ''
@@ -241,7 +167,7 @@ export default {
       }
     },
     onChangeNextStage (row) {
-      this.fields.forEach(f => {
+      this.list.forEach(f => {
         if (f.name !== row.name) {
           this.$set(f, 'next_stage', '')
         }
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 4d2883e1..e4659c54 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -166,6 +166,7 @@ export default {
   'Stage': '阶段',
   'Is List': '是否为列表',
   'Pagination': '分页',
+  'Settings': '设置',
 
   // 爬虫列表
   'Name': '名称',

From eb7d8b8fa34486faff6e70b6d56f73a4713f4f04 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Mon, 2 Dec 2019 13:29:00 +0800
Subject: [PATCH 38/52] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E8=AE=BE=E7=BD=AE=E9=83=A8=E5=88=86?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 68 +++++++++++++---
 .../components/TableView/FieldsTableView.vue  | 24 +++++-
 .../TableView/SettingFieldsTableView.vue      | 81 +++++++++++--------
 frontend/src/store/modules/spider.js          | 10 +++
 frontend/src/views/spider/SpiderDetail.vue    |  3 +
 5 files changed, 136 insertions(+), 50 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 2c730b3b..160de988 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -177,7 +177,7 @@
                     :value="isList(stage)"
                     @change="onCheckIsList($event, stage)"
                   />
-                  <el-popover v-if="isList(stage)" placement="top">
+                  <el-popover v-model="stage.isListOpen" v-if="isList(stage)" placement="top">
                     <div>
                       <el-tag :class="stage.list_css ? 'active' : 'inactive'" type="success"
                               @click="onSelectStageListType(stage, 'css')">CSS
@@ -190,11 +190,24 @@
                       <el-input v-if="stage.list_css" v-model="stage.list_css"/>
                       <el-input v-else v-model="stage.list_xpath"/>
                     </div>
-                    <el-tag v-if="stage.list_css" type="success" slot="reference"
-                            @click="onClickStageList($event, stage, 'css')">CSS
+                    <el-tag
+                      v-if="stage.list_css"
+                      type="success"
+                      slot="reference"
+                      @click="onClickStageList($event, stage, 'css')"
+                    >
+                      <i v-if="!stage.isListOpen" class="el-icon-circle-plus-outline"></i>
+                      <i v-else class="el-icon-remove-outline"></i>
+                      CSS
                     </el-tag>
-                    <el-tag v-else type="primary" slot="reference"
-                            @click="onClickStageList($event, stage, 'xpath')">
+                    <el-tag
+                      v-else
+                      type="primary"
+                      slot="reference"
+                      @click="onClickStageList($event, stage, 'xpath')"
+                    >
+                      <i v-if="!stage.isListOpen" class="el-icon-circle-plus-outline"></i>
+                      <i v-else class="el-icon-remove-outline"></i>
                       XPath
                     </el-tag>
                   </el-popover>
@@ -208,8 +221,9 @@
                     style="text-align: left; flex-basis: 20px; margin-right: 5px"
                     :value="isPage(stage)"
                     @change="onCheckIsPage($event, stage)"
+                    :disabled="!isList(stage)"
                   />
-                  <el-popover v-if="isPage(stage)" placement="top">
+                  <el-popover v-model="stage.isPageOpen" v-if="isPage(stage)" placement="top">
                     <div>
                       <el-tag :class="stage.page_css ? 'active' : 'inactive'" type="success"
                               @click="onSelectStagePageType(stage, 'css')">CSS
@@ -222,11 +236,24 @@
                       <el-input v-if="stage.page_css" v-model="stage.page_css"/>
                       <el-input v-else v-model="stage.page_xpath"/>
                     </div>
-                    <el-tag v-if="stage.page_css" type="success" slot="reference"
-                            @click="onClickStagePage($event, stage, 'css')">CSS
+                    <el-tag
+                      v-if="stage.page_css"
+                      type="success"
+                      slot="reference"
+                      @click="onClickStagePage($event, stage, 'css')"
+                    >
+                      <i v-if="!stage.isPageOpen" class="el-icon-circle-plus-outline"></i>
+                      <i v-else class="el-icon-remove-outline"></i>
+                      CSS
                     </el-tag>
-                    <el-tag v-else type="primary" slot="reference"
-                            @click="onClickStagePage($event, stage, 'xpath')">
+                    <el-tag
+                      v-else
+                      type="primary"
+                      slot="reference"
+                      @click="onClickStagePage($event, stage, 'xpath')"
+                    >
+                      <i v-if="!stage.isPageOpen" class="el-icon-circle-plus-outline"></i>
+                      <i v-else class="el-icon-remove-outline"></i>
                       XPath
                     </el-tag>
                   </el-popover>
@@ -255,7 +282,7 @@
       <!--Setting-->
       <el-tab-pane name="settings" :label="$t('Settings')">
         <div class="actions" style="text-align: right;margin-bottom: 10px">
-          <el-button type="success" size="small">
+          <el-button type="success" size="small" @click="onSave">
             {{$t('Save')}}
           </el-button>
         </div>
@@ -522,6 +549,25 @@ export default {
     },
     onClickSelectorType (selectorType) {
       Object.values(this.spiderForm.config.stages).forEach(stage => {
+        // 列表
+        if (selectorType === 'css') {
+          if (stage.list_xpath) stage.list_xpath = ''
+          if (!stage.list_css) stage.list_css = 'body'
+        } else {
+          if (stage.list_css) stage.list_css = ''
+          if (!stage.list_xpath) stage.list_xpath = '//body'
+        }
+
+        // 分页
+        if (selectorType === 'css') {
+          if (stage.page_xpath) stage.page_xpath = ''
+          if (!stage.page_css) stage.page_css = 'body'
+        } else {
+          if (stage.page_css) stage.page_css = ''
+          if (!stage.page_xpath) stage.page_xpath = '//body'
+        }
+
+        // 字段
         stage.fields.forEach(field => {
           if (selectorType === 'css') {
             if (field.xpath) field.xpath = ''
diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue
index 40c276de..2dbcfd81 100644
--- a/frontend/src/components/TableView/FieldsTableView.vue
+++ b/frontend/src/components/TableView/FieldsTableView.vue
@@ -23,6 +23,7 @@
           <template slot-scope="scope">
             <el-input v-model="scope.row.name"
                       :placeholder="$t('Field Name')"
+                      suffix-icon="el-icon-edit"
                       @change="onNameChange(scope.row)"
             />
           </template>
@@ -50,10 +51,20 @@
         <el-table-column :label="$t('Selector')" width="200px">
           <template slot-scope="scope">
             <template v-if="scope.row.css">
-              <el-input v-model="scope.row.css" :placeholder="$t('CSS / XPath')"></el-input>
+              <el-input
+                v-model="scope.row.css"
+                :placeholder="$t('CSS / XPath')"
+                suffix-icon="el-icon-edit"
+              >
+              </el-input>
             </template>
             <template v-else>
-              <el-input v-model="scope.row.xpath" :placeholder="$t('CSS / XPath')"></el-input>
+              <el-input
+                v-model="scope.row.xpath"
+                :placeholder="$t('CSS / XPath')"
+                suffix-icon="el-icon-edit"
+              >
+              </el-input>
             </template>
           </template>
         </el-table-column>
@@ -80,7 +91,12 @@
         <el-table-column :label="$t('Attribute')" width="200px">
           <template slot-scope="scope">
             <template v-if="isShowAttr(scope.row)">
-              <el-input v-model="scope.row.attr" :placeholder="$t('Attribute')" @change="onAttrChange(scope.row)"/>
+              <el-input
+                v-model="scope.row.attr"
+                :placeholder="$t('Attribute')"
+                suffix-icon="el-icon-edit"
+                @change="onAttrChange(scope.row)"
+              />
             </template>
             <template v-else>
               <span style="margin-left: 15px; color: lightgrey">
@@ -103,7 +119,7 @@
         </el-table-column>
         <el-table-column :label="$t('Remark')" width="auto" min-width="120px">
           <template slot-scope="scope">
-            <el-input v-model="scope.row.remark" :placeholder="$t('Remark')"/>
+            <el-input v-model="scope.row.remark" :placeholder="$t('Remark')" suffix-icon="el-icon-edit"/>
           </template>
         </el-table-column>
       </el-table>
diff --git a/frontend/src/components/TableView/SettingFieldsTableView.vue b/frontend/src/components/TableView/SettingFieldsTableView.vue
index 77caf85c..2a4ce331 100644
--- a/frontend/src/components/TableView/SettingFieldsTableView.vue
+++ b/frontend/src/components/TableView/SettingFieldsTableView.vue
@@ -1,11 +1,5 @@
 <template>
   <div class="setting-list-table-view">
-    <!--    <el-row class="button-group-container">-->
-    <!--      <label class="title">{{$t(this.title)}}</label>-->
-    <!--      <div class="button-group">-->
-    <!--        <el-button type="primary" size="small" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>-->
-    <!--      </div>-->
-    <!--    </el-row>-->
     <el-row>
       <el-table :data="list"
                 class="table edit"
@@ -14,22 +8,24 @@
       >
         <el-table-column class-name="action" width="80px" align="right">
           <template slot-scope="scope">
-            <i class="action-item el-icon-copy-document" @click="onCopyField(scope.row)"></i>
+            <!--            <i class="action-item el-icon-copy-document" @click="onCopyField(scope.row)"></i>-->
             <i class="action-item el-icon-remove-outline" @click="onRemoveField(scope.row)"></i>
             <i class="action-item el-icon-circle-plus-outline" @click="onAddField(scope.row)"></i>
           </template>
         </el-table-column>
-        <el-table-column :label="$t('Name')" width="150px">
+        <el-table-column :label="$t('Name')" width="240px">
           <template slot-scope="scope">
-            <el-input v-model="scope.row.name"
-                      :placeholder="$t('Name')"
-                      @change="onNameChange(scope.row)"
+            <el-input
+              v-model="scope.row.name"
+              :placeholder="$t('Name')"
+              suffix-icon="el-icon-edit"
+              @change="onNameChange(scope.row)"
             />
           </template>
         </el-table-column>
-        <el-table-column :label="$t('Remark')" width="auto" min-width="120px">
+        <el-table-column :label="$t('Value')" width="auto" min-width="120px">
           <template slot-scope="scope">
-            <el-input v-model="scope.row.remark" :placeholder="$t('Remark')"/>
+            <el-input v-model="scope.row.value" :placeholder="$t('Value')" suffix-icon="el-icon-edit"/>
           </template>
         </el-table-column>
       </el-table>
@@ -58,18 +54,22 @@ export default {
       default () {
         return []
       }
-    },
-    list: {
-      type: Array,
-      default () {
-        return []
-      }
     }
   },
   computed: {
     ...mapState('spider', [
       'spiderForm'
-    ])
+    ]),
+    list () {
+      const list = []
+      for (let name in this.spiderForm.config.settings) {
+        if (this.spiderForm.config.settings.hasOwnProperty(name)) {
+          const value = this.spiderForm.config.settings[name]
+          list.push({ name, value })
+        }
+      }
+      return list
+    }
   },
   methods: {
     addField () {
@@ -124,27 +124,29 @@ export default {
       }
     },
     onRemoveField (row) {
-      for (let i = 0; i < this.list.length; i++) {
-        if (row.name === this.list[i].name) {
-          this.list.splice(i, 1)
-          break
-        }
-      }
+      console.log(row)
+      this.$store.commit(
+        'spider/UNSET_SPIDER_FORM_CONFIG_SETTING_ITEM',
+        row.name
+      )
       if (this.list.length === 0) {
-        this.list.push({
-          css: 'body',
-          next_stage: ''
-        })
+        this.$store.commit(
+          'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM',
+          'VARIABLE_NAME_' + Math.floor(new Date().getTime()),
+          'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
+        )
       }
     },
     onAddField (row) {
       for (let i = 0; i < this.list.length; i++) {
         if (row.name === this.list[i].name) {
-          this.list.splice(i + 1, 0, {
-            name: `field_${Math.floor(new Date().getTime()).toString()}`,
-            css: 'body',
-            next_stage: ''
-          })
+          this.$store.commit(
+            'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM',
+            {
+              name: 'VARIABLE_NAME_' + Math.floor(new Date().getTime()),
+              value: 'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
+            }
+          )
           break
         }
       }
@@ -173,6 +175,15 @@ export default {
         }
       })
     }
+  },
+  created () {
+    if (this.list.length === 0) {
+      this.$store.commit(
+        'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM',
+        'VARIABLE_NAME_' + Math.floor(new Date().getTime()),
+        'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
+      )
+    }
   }
 }
 </script>
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index e653db3d..161057a5 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -1,3 +1,4 @@
+import Vue from 'vue'
 import request from '../../api/request'
 
 const state = {
@@ -72,6 +73,15 @@ const mutations = {
   },
   SET_PREVIEW_CRAWL_DATA (state, value) {
     state.previewCrawlData = value
+  },
+  SET_SPIDER_FORM_CONFIG_SETTING_ITEM (state, payload) {
+    const { name, value } = payload
+    Vue.set(state.spiderForm.config.settings, name, value)
+  },
+  UNSET_SPIDER_FORM_CONFIG_SETTING_ITEM (state, name) {
+    const settings = JSON.parse(JSON.stringify(state.spiderForm.config.settings))
+    delete settings[name]
+    Vue.set(state.spiderForm.config, 'settings', settings)
   }
 }
 
diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue
index 4986957a..fd85921f 100644
--- a/frontend/src/views/spider/SpiderDetail.vue
+++ b/frontend/src/views/spider/SpiderDetail.vue
@@ -109,6 +109,9 @@ export default {
 
     // get spider tasks
     this.$store.dispatch('spider/getTaskList', this.$route.params.id)
+
+    // get spider list
+    this.$store.dispatch('spider/getSpiderList')
   }
 }
 </script>

From bca50c0f19c3267d9d79fda7cdcd3cf5893522ab Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Mon, 2 Dec 2019 17:46:57 +0800
Subject: [PATCH 39/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E4=BE=9D=E8=B5=96?=
 =?UTF-8?q?=E6=A8=A1=E5=9D=97?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/template/scrapy/config_spider/settings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/template/scrapy/config_spider/settings.py b/backend/template/scrapy/config_spider/settings.py
index 3dde4c6c..c5cb4a1e 100644
--- a/backend/template/scrapy/config_spider/settings.py
+++ b/backend/template/scrapy/config_spider/settings.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import os
 
 # Scrapy settings for config_spider project
 #

From c3bc20127dd183df6aa01ff08b08a9a49538a582 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Mon, 2 Dec 2019 22:35:45 +0800
Subject: [PATCH 40/52] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../template/scrapy/config_spider/settings.py | 17 ++++--
 .../TableView/SettingFieldsTableView.vue      | 53 ++++++++-----------
 frontend/src/store/modules/spider.js          | 12 ++---
 3 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/backend/template/scrapy/config_spider/settings.py b/backend/template/scrapy/config_spider/settings.py
index c5cb4a1e..fab0080f 100644
--- a/backend/template/scrapy/config_spider/settings.py
+++ b/backend/template/scrapy/config_spider/settings.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import os
+import re
 
 # Scrapy settings for config_spider project
 #
@@ -17,7 +18,7 @@ NEWSPIDER_MODULE = 'config_spider.spiders'
 
 
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = 'config_spider (+http://www.yourdomain.com)'
+USER_AGENT = 'Crawlab Spider'
 
 # Obey robots.txt rules
 ROBOTSTXT_OBEY = True
@@ -90,8 +91,18 @@ ITEM_PIPELINES = {
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 
-for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWALAB_SETTING_')]:
+for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWLAB_SETTING_')]:
     setting_name = setting_env_name.replace('CRAWLAB_SETTING_', '')
-    setting_value = os.environ.get('setting_env_name')
+    setting_value = os.environ.get(setting_env_name)
+    if setting_value.lower() == 'true':
+        setting_value = True
+    elif setting_value.lower() == 'false':
+        setting_value = False
+    elif re.search(r'^\d+$', setting_value) is not None:
+        setting_value = int(setting_value)
+    else:
+        pass
+    print(setting_name)
+    print(setting_value)
     locals()[setting_name] = setting_value
 
diff --git a/frontend/src/components/TableView/SettingFieldsTableView.vue b/frontend/src/components/TableView/SettingFieldsTableView.vue
index 2a4ce331..701f8ef5 100644
--- a/frontend/src/components/TableView/SettingFieldsTableView.vue
+++ b/frontend/src/components/TableView/SettingFieldsTableView.vue
@@ -19,13 +19,18 @@
               v-model="scope.row.name"
               :placeholder="$t('Name')"
               suffix-icon="el-icon-edit"
-              @change="onNameChange(scope.row)"
+              @change="onChange(scope.row)"
             />
           </template>
         </el-table-column>
         <el-table-column :label="$t('Value')" width="auto" min-width="120px">
           <template slot-scope="scope">
-            <el-input v-model="scope.row.value" :placeholder="$t('Value')" suffix-icon="el-icon-edit"/>
+            <el-input
+              v-model="scope.row.value"
+              :placeholder="$t('Value')"
+              suffix-icon="el-icon-edit"
+              @change="onChange(scope.row)"
+            />
           </template>
         </el-table-column>
       </el-table>
@@ -83,10 +88,11 @@ export default {
       this.list.splice(index, 1)
       this.$st.sendEv('爬虫详情-配置', '删除字段')
     },
-    onNameChange (row) {
+    onChange (row) {
       if (this.list.filter(d => d.name === row.name).length > 1) {
         this.$message.error(this.$t(`Duplicated field names for ${row.name}`))
       }
+      this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', this.list)
       this.$st.sendEv('爬虫详情-配置', '更改字段')
     },
     onCheck (row) {
@@ -115,41 +121,26 @@ export default {
         if (!row.attr) this.$set(row, 'attr', 'href')
       }
     },
-    onCopyField (row) {
-      for (let i = 0; i < this.list.length; i++) {
-        if (row.name === this.list[i].name) {
-          this.list.splice(i, 0, JSON.parse(JSON.stringify(row)))
-          break
+    onRemoveField (row) {
+      const list = JSON.parse(JSON.stringify(this.list))
+      for (let i = 0; i < list.length; i++) {
+        if (row.name === list[i].name) {
+          list.splice(i, 0, 1)
         }
       }
-    },
-    onRemoveField (row) {
-      console.log(row)
-      this.$store.commit(
-        'spider/UNSET_SPIDER_FORM_CONFIG_SETTING_ITEM',
-        row.name
-      )
-      if (this.list.length === 0) {
-        this.$store.commit(
-          'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM',
-          'VARIABLE_NAME_' + Math.floor(new Date().getTime()),
-          'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
-        )
-      }
+      this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', list)
     },
     onAddField (row) {
-      for (let i = 0; i < this.list.length; i++) {
-        if (row.name === this.list[i].name) {
-          this.$store.commit(
-            'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM',
-            {
-              name: 'VARIABLE_NAME_' + Math.floor(new Date().getTime()),
-              value: 'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
-            }
-          )
+      const list = JSON.parse(JSON.stringify(this.list))
+      for (let i = 0; i < list.length; i++) {
+        if (row.name === list[i].name) {
+          const name = 'VARIABLE_NAME_' + Math.floor(new Date().getTime())
+          const value = 'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
+          list.push({ name, value })
           break
         }
       }
+      this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', list)
     },
     getCellClassStyle ({ row, columnIndex }) {
       if (columnIndex === 1) {
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index 161057a5..10c4657e 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -74,13 +74,11 @@ const mutations = {
   SET_PREVIEW_CRAWL_DATA (state, value) {
     state.previewCrawlData = value
   },
-  SET_SPIDER_FORM_CONFIG_SETTING_ITEM (state, payload) {
-    const { name, value } = payload
-    Vue.set(state.spiderForm.config.settings, name, value)
-  },
-  UNSET_SPIDER_FORM_CONFIG_SETTING_ITEM (state, name) {
-    const settings = JSON.parse(JSON.stringify(state.spiderForm.config.settings))
-    delete settings[name]
+  SET_SPIDER_FORM_CONFIG_SETTINGS (state, payload) {
+    const settings = {}
+    payload.forEach(row => {
+      settings[row.name] = row.value
+    })
     Vue.set(state.spiderForm.config, 'settings', settings)
   }
 }

From 9e849695d1f2cf2b9678cab2e544b94ae2e88a40 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 3 Dec 2019 12:14:38 +0800
Subject: [PATCH 41/52] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/config_spider/scrapy.go             | 2 +-
 backend/template/scrapy/config_spider/settings.py | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/backend/model/config_spider/scrapy.go b/backend/model/config_spider/scrapy.go
index 5a6730d1..5f4870e4 100644
--- a/backend/model/config_spider/scrapy.go
+++ b/backend/model/config_spider/scrapy.go
@@ -187,7 +187,7 @@ func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stag
 	// 分页
 	if stage.PageCss != "" || stage.PageXpath != "" {
 		str += g.PadCode(fmt.Sprintf(`next_url = response.%s.extract_first()`, g.GetExtractStringFromStage(stage)), 2)
-		str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': item})`, stageName), 2)
+		str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': prev_item})`, stageName), 2)
 	}
 
 	// 加入末尾换行
diff --git a/backend/template/scrapy/config_spider/settings.py b/backend/template/scrapy/config_spider/settings.py
index fab0080f..195f95df 100644
--- a/backend/template/scrapy/config_spider/settings.py
+++ b/backend/template/scrapy/config_spider/settings.py
@@ -102,7 +102,5 @@ for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWLAB_S
         setting_value = int(setting_value)
     else:
         pass
-    print(setting_name)
-    print(setting_value)
     locals()[setting_name] = setting_value
 

From 98cbccb81e96efe5106807b34345c7f2985da5e0 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 3 Dec 2019 13:37:41 +0800
Subject: [PATCH 42/52] =?UTF-8?q?=E5=8A=A0=E5=85=A5Spiderfile=E6=A8=A1?=
 =?UTF-8?q?=E7=89=88?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/main.go                               |  1 +
 backend/model/spider.go                       |  3 +
 backend/routes/config_spider.go               | 28 ++++++++-
 .../template/scrapy/config_spider/settings.py |  5 ++
 .../template/spiderfile/Spiderfile.163_news   | 20 +++++++
 backend/template/spiderfile/Spiderfile.baidu  | 22 +++++++
 .../Spiderfile.toscrapy_books}                |  5 +-
 frontend/src/components/Config/ConfigList.vue | 58 ++++++++++---------
 .../TableView/SettingFieldsTableView.vue      |  8 ++-
 frontend/src/i18n/zh.js                       |  3 +
 frontend/src/store/modules/spider.js          | 13 ++++-
 frontend/src/views/spider/SpiderDetail.vue    | 20 ++++---
 frontend/src/views/spider/SpiderList.vue      | 43 ++++++++++----
 13 files changed, 177 insertions(+), 52 deletions(-)
 create mode 100644 backend/template/spiderfile/Spiderfile.163_news
 create mode 100644 backend/template/spiderfile/Spiderfile.baidu
 rename backend/template/{Spiderfile => spiderfile/Spiderfile.toscrapy_books} (88%)

diff --git a/backend/main.go b/backend/main.go
index 226b3bd5..b49efae7 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -146,6 +146,7 @@ func main() {
 			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)                      // 修改可配置爬虫
 			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider)             // 上传可配置爬虫
 			authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
+			authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList)      // 获取可配置爬虫模版列表
 			// 任务
 			authGroup.GET("/tasks", routes.GetTaskList)                                 // 任务列表
 			authGroup.GET("/tasks/:id", routes.GetTask)                                 // 任务详情
diff --git a/backend/model/spider.go b/backend/model/spider.go
index a06e682b..a0d72c1c 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -36,6 +36,9 @@ type Spider struct {
 	// 自定义爬虫
 	Cmd string `json:"cmd" bson:"cmd"` // 执行命令
 
+	// 可配置爬虫
+	Template string `json:"template" bson:"template"` // Spiderfile模版
+
 	// 前端展示
 	LastRunTs  time.Time               `json:"last_run_ts"` // 最后一次执行时间
 	LastStatus string                  `json:"last_status"` // 最后执行状态
diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 61067f98..3e0f0e56 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -16,6 +16,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // 添加可配置爬虫
@@ -32,6 +33,12 @@ func PutConfigSpider(c *gin.Context) {
 		return
 	}
 
+	// 模版名不能为空
+	if spider.Template == "" {
+		HandleErrorF(http.StatusBadRequest, c, "spider template should not be empty")
+		return
+	}
+
 	// 判断爬虫是否存在
 	if spider := model.GetSpiderByName(spider.Name); spider != nil {
 		HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("spider for '%s' already exists", spider.Name))
@@ -59,7 +66,7 @@ func PutConfigSpider(c *gin.Context) {
 	spider.Src = spiderDir
 
 	// 复制Spiderfile模版
-	contentByte, err := ioutil.ReadFile("./template/Spiderfile")
+	contentByte, err := ioutil.ReadFile("./template/spiderfile/Spiderfile." + spider.Template)
 	if err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return
@@ -113,8 +120,8 @@ func UploadConfigSpider(c *gin.Context) {
 
 	// 文件名称必须为Spiderfile
 	filename := header.Filename
-	if filename != "Spiderfile" {
-		HandleErrorF(http.StatusBadRequest, c, "filename must be 'Spiderfile'")
+	if filename != "Spiderfile" && filename != "Spiderfile.yaml" && filename != "Spiderfile.yml" {
+		HandleErrorF(http.StatusBadRequest, c, "filename must be 'Spiderfile(.yaml|.yml)'")
 		return
 	}
 
@@ -286,3 +293,18 @@ func GetConfigSpiderConfig(c *gin.Context) {
 		Data:    spider.Config,
 	})
 }
+
+// 获取模版名称列表
+func GetConfigSpiderTemplateList(c *gin.Context) {
+	var data []string
+	for _, fInfo := range utils.ListDir("./template/spiderfile") {
+		templateName := strings.Replace(fInfo.Name(), "Spiderfile.", "", -1)
+		data = append(data, templateName)
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    data,
+	})
+}
diff --git a/backend/template/scrapy/config_spider/settings.py b/backend/template/scrapy/config_spider/settings.py
index 195f95df..4b0965f2 100644
--- a/backend/template/scrapy/config_spider/settings.py
+++ b/backend/template/scrapy/config_spider/settings.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import os
 import re
+import json
 
 # Scrapy settings for config_spider project
 #
@@ -100,6 +101,10 @@ for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWLAB_S
         setting_value = False
     elif re.search(r'^\d+$', setting_value) is not None:
         setting_value = int(setting_value)
+    elif re.search(r'^\{.*\}$', setting_value.strip()) is not None:
+        setting_value = json.loads(setting_value)
+    elif re.search(r'^\[.*\]$', setting_value.strip()) is not None:
+        setting_value = json.loads(setting_value)
     else:
         pass
     locals()[setting_name] = setting_value
diff --git a/backend/template/spiderfile/Spiderfile.163_news b/backend/template/spiderfile/Spiderfile.163_news
new file mode 100644
index 00000000..29d58279
--- /dev/null
+++ b/backend/template/spiderfile/Spiderfile.163_news
@@ -0,0 +1,20 @@
+version: "0.4.0"
+name: "toscrapy_books"
+start_url: "http://news.163.com/special/0001386F/rank_news.html"
+start_stage: "list"
+engine: "scrapy"
+stages:
+  list:
+    is_list: true
+    list_css: "table tr:not(:first-child)"
+    fields:
+      - name: "title"
+        css: "td:nth-child(1) > a"
+      - name: "url"
+        css: "td:nth-child(1) > a"
+        attr: "href"
+      - name: "clicks"
+        css: "td.cBlue"
+settings:
+  ROBOTSTXT_OBEY: false
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
diff --git a/backend/template/spiderfile/Spiderfile.baidu b/backend/template/spiderfile/Spiderfile.baidu
new file mode 100644
index 00000000..86388621
--- /dev/null
+++ b/backend/template/spiderfile/Spiderfile.baidu
@@ -0,0 +1,22 @@
+version: "0.4.0"
+name: "toscrapy_books"
+start_url: "http://www.baidu.com/s?wd=crawlab"
+start_stage: "list"
+engine: "scrapy"
+stages:
+  list:
+    is_list: true
+    list_css: ".result.c-container"
+    page_css: "#page a.n:last-child"
+    page_attr: "href"
+    fields:
+      - name: "title"
+        css: "h3 > a"
+      - name: "url"
+        css: "h3 > a"
+        attr: "href"
+      - name: "abstract"
+        css: ".c-abstract"
+settings:
+  ROBOTSTXT_OBEY: false
+  USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
diff --git a/backend/template/Spiderfile b/backend/template/spiderfile/Spiderfile.toscrapy_books
similarity index 88%
rename from backend/template/Spiderfile
rename to backend/template/spiderfile/Spiderfile.toscrapy_books
index d748d5f8..4bf18f61 100644
--- a/backend/template/Spiderfile
+++ b/backend/template/spiderfile/Spiderfile.toscrapy_books
@@ -5,10 +5,10 @@ start_stage: "list"
 engine: "scrapy"
 stages:
   list:
-    is_list: true # default: false
+    is_list: true
     list_css: "section article.product_pod"
     page_css: "ul.pager li.next a"
-    page_attr: "href" # default: href
+    page_attr: "href"
     fields:
       - name: "title"
         css: "h3 > a"
@@ -25,3 +25,4 @@ stages:
         css: "#product_description + p"
 settings:
   ROBOTSTXT_OBEY: true
+  AUTOTHROTTLE_ENABLED: true
diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 160de988..1fc62d18 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -171,25 +171,27 @@
 
                 <!--list-->
                 <li class="stage-item" style="min-width: 240px">
-                  <label>{{$t('Is List')}}: </label>
+                  <label>{{$t('List')}}: </label>
                   <el-checkbox
                     style="text-align: left; flex-basis: 20px; margin-right: 5px"
                     :value="isList(stage)"
                     @change="onCheckIsList($event, stage)"
                   />
-                  <el-popover v-model="stage.isListOpen" v-if="isList(stage)" placement="top">
-                    <div>
-                      <el-tag :class="stage.list_css ? 'active' : 'inactive'" type="success"
-                              @click="onSelectStageListType(stage, 'css')">CSS
-                      </el-tag>
-                      <el-tag :class="!stage.list_css ? 'active' : 'inactive'" type="primary"
-                              @click="onSelectStageListType(stage, 'xpath')">XPath
-                      </el-tag>
-                    </div>
-                    <div class="list-selector" style="margin-top: 5px; width: 240px">
-                      <el-input v-if="stage.list_css" v-model="stage.list_css"/>
-                      <el-input v-else v-model="stage.list_xpath"/>
-                    </div>
+                  <el-popover v-model="stage.isListOpen" v-if="isList(stage)" placement="top" width="360">
+                    <el-form label-width="120px">
+                      <el-form-item :label="$t('Selector Type')">
+                        <el-tag :class="stage.list_css ? 'active' : 'inactive'" type="success"
+                                @click="onSelectStageListType(stage, 'css')">CSS
+                        </el-tag>
+                        <el-tag :class="!stage.list_css ? 'active' : 'inactive'" type="primary"
+                                @click="onSelectStageListType(stage, 'xpath')">XPath
+                        </el-tag>
+                      </el-form-item>
+                      <el-form-item :label="$t('Selector')" class="list-selector">
+                        <el-input v-if="stage.list_css" v-model="stage.list_css"/>
+                        <el-input v-else v-model="stage.list_xpath"/>
+                      </el-form-item>
+                    </el-form>
                     <el-tag
                       v-if="stage.list_css"
                       type="success"
@@ -223,19 +225,21 @@
                     @change="onCheckIsPage($event, stage)"
                     :disabled="!isList(stage)"
                   />
-                  <el-popover v-model="stage.isPageOpen" v-if="isPage(stage)" placement="top">
-                    <div>
-                      <el-tag :class="stage.page_css ? 'active' : 'inactive'" type="success"
-                              @click="onSelectStagePageType(stage, 'css')">CSS
-                      </el-tag>
-                      <el-tag :class="!stage.page_css ? 'active' : 'inactive'" type="primary"
-                              @click="onSelectStagePageType(stage, 'xpath')">XPath
-                      </el-tag>
-                    </div>
-                    <div class="page-selector" style="margin-top: 5px; width: 240px">
-                      <el-input v-if="stage.page_css" v-model="stage.page_css"/>
-                      <el-input v-else v-model="stage.page_xpath"/>
-                    </div>
+                  <el-popover v-model="stage.isPageOpen" v-if="isPage(stage)" placement="top" width="360">
+                    <el-form label-width="120px">
+                      <el-form-item :label="$t('Selector Type')">
+                        <el-tag :class="stage.page_css ? 'active' : 'inactive'" type="success"
+                                @click="onSelectStagePageType(stage, 'css')">CSS
+                        </el-tag>
+                        <el-tag :class="!stage.page_css ? 'active' : 'inactive'" type="primary"
+                                @click="onSelectStagePageType(stage, 'xpath')">XPath
+                        </el-tag>
+                      </el-form-item>
+                      <el-form-item :label="$t('Selector')" class="page-selector">
+                        <el-input v-if="stage.page_css" v-model="stage.page_css"/>
+                        <el-input v-else v-model="stage.page_xpath"/>
+                      </el-form-item>
+                    </el-form>
                     <el-tag
                       v-if="stage.page_css"
                       type="success"
diff --git a/frontend/src/components/TableView/SettingFieldsTableView.vue b/frontend/src/components/TableView/SettingFieldsTableView.vue
index 701f8ef5..7ce8a46c 100644
--- a/frontend/src/components/TableView/SettingFieldsTableView.vue
+++ b/frontend/src/components/TableView/SettingFieldsTableView.vue
@@ -125,9 +125,15 @@ export default {
       const list = JSON.parse(JSON.stringify(this.list))
       for (let i = 0; i < list.length; i++) {
         if (row.name === list[i].name) {
-          list.splice(i, 0, 1)
+          list.splice(i, 1)
         }
       }
+      if (list.length === 0) {
+        list.push({
+          name: `VARIABLE_NAME_${Math.floor(new Date().getTime())}`,
+          value: `VARIABLE_VALUE_${Math.floor(new Date().getTime())}`
+        })
+      }
       this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', list)
     },
     onAddField (row) {
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index e4659c54..c7d350d0 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -165,8 +165,11 @@ export default {
   'Fields': '字段',
   'Stage': '阶段',
   'Is List': '是否为列表',
+  'List': '列表',
   'Pagination': '分页',
   'Settings': '设置',
+  'Display Name': '显示名称',
+  'Template': '模版',
 
   // 爬虫列表
   'Name': '名称',
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index 10c4657e..0b4c2724 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -1,5 +1,6 @@
 import Vue from 'vue'
 import request from '../../api/request'
+import axisModelCommonMixin from 'echarts/src/coord/axisModelCommonMixin'
 
 const state = {
   // list of spiders
@@ -35,7 +36,10 @@ const state = {
   filterSite: '',
 
   // preview crawl data
-  previewCrawlData: []
+  previewCrawlData: [],
+
+  // template list
+  templateList: []
 }
 
 const getters = {}
@@ -80,6 +84,9 @@ const mutations = {
       settings[row.name] = row.value
     })
     Vue.set(state.spiderForm.config, 'settings', settings)
+  },
+  SET_TEMPLATE_LIST (state, value) {
+    state.templateList = value
   }
 }
 
@@ -166,6 +173,10 @@ const actions = {
   },
   addConfigSpider ({ state }) {
     return request.put(`/config_spiders`, state.spiderForm)
+  },
+  async getTemplateList ({ state, commit }) {
+    const res = await request.get(`/config_spiders_templates`)
+    commit('SET_TEMPLATE_LIST', res.data.data)
   }
 }
 
diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue
index fd85921f..a743e47d 100644
--- a/frontend/src/views/spider/SpiderDetail.vue
+++ b/frontend/src/views/spider/SpiderDetail.vue
@@ -96,22 +96,26 @@ export default {
       this.$st.sendEv('爬虫详情', '切换爬虫')
     }
   },
-  created () {
+  async created () {
     // get the list of the spiders
     // this.$store.dispatch('spider/getSpiderList')
 
     // get spider basic info
-    this.$store.dispatch('spider/getSpiderData', this.$route.params.id)
-      .then(() => {
-        // get spider file info
-        this.$store.dispatch('file/getFileList', this.spiderForm.src)
-      })
+    await this.$store.dispatch('spider/getSpiderData', this.$route.params.id)
+
+    // get spider file info
+    await this.$store.dispatch('file/getFileList', this.spiderForm.src)
 
     // get spider tasks
-    this.$store.dispatch('spider/getTaskList', this.$route.params.id)
+    await this.$store.dispatch('spider/getTaskList', this.$route.params.id)
 
     // get spider list
-    this.$store.dispatch('spider/getSpiderList')
+    await this.$store.dispatch('spider/getSpiderList')
+
+    // if spider is configurable spider, set to config tab by default
+    if (this.spiderForm.type === 'configurable') {
+      this.activeTabName = 'config'
+    }
   }
 }
 </script>
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index c756ce52..cc3d6acb 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -42,6 +42,16 @@
             <el-form-item :label="$t('Display Name')" prop="display_name" required>
               <el-input v-model="spiderForm.display_name" :placeholder="$t('Display Name')"/>
             </el-form-item>
+            <el-form-item :label="$t('Template')" prop="template" required>
+              <el-select v-model="spiderForm.template" :value="spiderForm.template" :placeholder="$t('Template')">
+                <el-option
+                  v-for="template in templateList"
+                  :key="template"
+                  :label="template"
+                  :value="template"
+                />
+              </el-select>
+            </el-form-item>
             <el-form-item :label="$t('Results')" prop="col" required>
               <el-input v-model="spiderForm.col" :placeholder="$t('Results')"/>
             </el-form-item>
@@ -319,7 +329,8 @@ export default {
       'importForm',
       'spiderList',
       'spiderForm',
-      'spiderTotal'
+      'spiderTotal',
+      'templateList'
     ]),
     ...mapGetters('user', [
       'token'
@@ -342,7 +353,9 @@ export default {
       this.getList()
     },
     onAdd () {
-      this.$store.commit('spider/SET_SPIDER_FORM', {})
+      this.$store.commit('spider/SET_SPIDER_FORM', {
+        template: this.templateList[0]
+      })
       this.addDialogVisible = true
     },
     onAddConfigurable () {
@@ -535,19 +548,29 @@ export default {
         type: this.filter.type
       }
       this.$store.dispatch('spider/getSpiderList', params)
-    },
-    getTypes () {
-      request.get(`/spider/types`).then(resp => {
-        this.types = resp.data.data
-      })
     }
+    // getTypes () {
+    //   request.get(`/spider/types`).then(resp => {
+    //     this.types = resp.data.data
+    //   })
+    // }
   },
-  created () {
-    this.getTypes()
+  async created () {
+    // fetch spider types
+    // await this.getTypes()
+
     // fetch spider list
-    this.getList()
+    await this.getList()
+
+    // fetch template list
+    await this.$store.dispatch('spider/getTemplateList')
   },
   mounted () {
+    console.log(this.spiderForm)
+    const vm = this
+    this.$nextTick(() => {
+      vm.$store.commit('spider/SET_SPIDER_FORM', this.spiderForm)
+    })
   }
 }
 </script>

From 40f6675ce893b9dd54fed0ec8e6432ef41789584 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 3 Dec 2019 14:25:54 +0800
Subject: [PATCH 43/52] =?UTF-8?q?=E6=9B=B4=E6=96=B0Spiderfile=E6=A8=A1?=
 =?UTF-8?q?=E7=89=88?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/config_spider/scrapy.go        | 2 +-
 backend/template/spiderfile/Spiderfile.baidu | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/backend/model/config_spider/scrapy.go b/backend/model/config_spider/scrapy.go
index 5f4870e4..bf52d42a 100644
--- a/backend/model/config_spider/scrapy.go
+++ b/backend/model/config_spider/scrapy.go
@@ -225,7 +225,7 @@ func (g ScrapyGenerator) GetExtractStringFromField(f entity.Field) string {
 		// 如果为XPath
 		if f.Attr == "" {
 			// 文本
-			return fmt.Sprintf(`xpath('%s/text()')`, f.Xpath)
+			return fmt.Sprintf(`xpath('string(%s)')`, f.Xpath)
 		} else {
 			// 属性
 			return fmt.Sprintf(`xpath('%s/@%s')`, f.Xpath, f.Attr)
diff --git a/backend/template/spiderfile/Spiderfile.baidu b/backend/template/spiderfile/Spiderfile.baidu
index 86388621..c97e87af 100644
--- a/backend/template/spiderfile/Spiderfile.baidu
+++ b/backend/template/spiderfile/Spiderfile.baidu
@@ -11,9 +11,9 @@ stages:
     page_attr: "href"
     fields:
       - name: "title"
-        css: "h3 > a"
+        xpath: ".//h3/a"
       - name: "url"
-        css: "h3 > a"
+        xpath: ".//h3/a"
         attr: "href"
       - name: "abstract"
         css: ".c-abstract"

From 6d9c162e9e372d8753e676919f5ab090139bd2b2 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 4 Dec 2019 13:57:27 +0800
Subject: [PATCH 44/52] =?UTF-8?q?=E4=BC=98=E5=85=88=E8=B0=83=E6=95=B4xpath?=
 =?UTF-8?q?=E9=A1=BA=E5=BA=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/config_spider/scrapy.go         | 10 ++++++-
 backend/services/config_spider.go             |  4 +--
 backend/template/spiderfile/Spiderfile.baidu  | 30 +++++++++----------
 frontend/src/components/Config/ConfigList.vue | 26 +++++++++-------
 .../components/TableView/FieldsTableView.vue  |  4 +--
 5 files changed, 43 insertions(+), 31 deletions(-)

diff --git a/backend/model/config_spider/scrapy.go b/backend/model/config_spider/scrapy.go
index bf52d42a..6fcb77f0 100644
--- a/backend/model/config_spider/scrapy.go
+++ b/backend/model/config_spider/scrapy.go
@@ -158,7 +158,7 @@ func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stag
 	str += g.PadCode(`prev_item = response.meta.get('item')`, 2)
 
 	// for 循环遍历列表
-	str += g.PadCode(fmt.Sprintf(`for elem in response.css('%s'):`, stage.ListCss), 2)
+	str += g.PadCode(fmt.Sprintf(`for elem in response.%s:`, g.GetListString(stage)), 2)
 
 	// 构造item
 	str += g.PadCode(`item = Item()`, 3)
@@ -248,3 +248,11 @@ func (g ScrapyGenerator) GetExtractStringFromStage(stage entity.Stage) string {
 		return fmt.Sprintf(`xpath('%s/@%s')`, stage.PageXpath, pageAttr)
 	}
 }
+
+func (g ScrapyGenerator) GetListString(stage entity.Stage) string {
+	if stage.ListCss != "" {
+		return fmt.Sprintf(`css('%s')`, stage.ListCss)
+	} else {
+		return fmt.Sprintf(`xpath('%s')`, stage.ListXpath)
+	}
+}
diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go
index d96146f5..adce0531 100644
--- a/backend/services/config_spider.go
+++ b/backend/services/config_spider.go
@@ -118,8 +118,8 @@ func ValidateSpiderfile(configData entity.ConfigSpiderData) error {
 		}
 
 		// 如果 stage 的 is_list 为 true 但 list_css 为空，报错
-		if stage.IsList && stage.ListCss == "" {
-			return errors.New("spiderfile invalid: stage with is_list = true should have list_css being set")
+		if stage.IsList && (stage.ListCss == "" && stage.ListXpath == "") {
+			return errors.New("spiderfile invalid: stage with is_list = true should have either list_css or list_xpath being set")
 		}
 	}
 
diff --git a/backend/template/spiderfile/Spiderfile.baidu b/backend/template/spiderfile/Spiderfile.baidu
index c97e87af..fbf720e4 100644
--- a/backend/template/spiderfile/Spiderfile.baidu
+++ b/backend/template/spiderfile/Spiderfile.baidu
@@ -1,22 +1,22 @@
-version: "0.4.0"
-name: "toscrapy_books"
-start_url: "http://www.baidu.com/s?wd=crawlab"
-start_stage: "list"
-engine: "scrapy"
+version: 0.4.0
+name: toscrapy_books
+start_url: http://www.baidu.com/s?wd=crawlab
+start_stage: list
+engine: scrapy
 stages:
   list:
     is_list: true
-    list_css: ".result.c-container"
-    page_css: "#page a.n:last-child"
-    page_attr: "href"
+    list_xpath: //*[contains(@class, "c-container")]
+    page_xpath: //*[@id="page"]//a[@class="n"][last()]
+    page_attr: href
     fields:
-      - name: "title"
-        xpath: ".//h3/a"
-      - name: "url"
-        xpath: ".//h3/a"
-        attr: "href"
-      - name: "abstract"
-        css: ".c-abstract"
+      - name: title
+        xpath: .//h3/a
+      - name: url
+        xpath: .//h3/a
+        attr: href
+      - name: abstract
+        xpath: .//*[@class="c-abstract"]
 settings:
   ROBOTSTXT_OBEY: false
   USER_AGENT: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 1fc62d18..bd31441e 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -180,20 +180,20 @@
                   <el-popover v-model="stage.isListOpen" v-if="isList(stage)" placement="top" width="360">
                     <el-form label-width="120px">
                       <el-form-item :label="$t('Selector Type')">
-                        <el-tag :class="stage.list_css ? 'active' : 'inactive'" type="success"
+                        <el-tag :class="!stage.list_xpath ? 'active' : 'inactive'" type="success"
                                 @click="onSelectStageListType(stage, 'css')">CSS
                         </el-tag>
-                        <el-tag :class="!stage.list_css ? 'active' : 'inactive'" type="primary"
+                        <el-tag :class="stage.list_xpath ? 'active' : 'inactive'" type="primary"
                                 @click="onSelectStageListType(stage, 'xpath')">XPath
                         </el-tag>
                       </el-form-item>
                       <el-form-item :label="$t('Selector')" class="list-selector">
-                        <el-input v-if="stage.list_css" v-model="stage.list_css"/>
+                        <el-input v-if="!stage.list_xpath" v-model="stage.list_css"/>
                         <el-input v-else v-model="stage.list_xpath"/>
                       </el-form-item>
                     </el-form>
                     <el-tag
-                      v-if="stage.list_css"
+                      v-if="!stage.list_xpath"
                       type="success"
                       slot="reference"
                       @click="onClickStageList($event, stage, 'css')"
@@ -228,20 +228,20 @@
                   <el-popover v-model="stage.isPageOpen" v-if="isPage(stage)" placement="top" width="360">
                     <el-form label-width="120px">
                       <el-form-item :label="$t('Selector Type')">
-                        <el-tag :class="stage.page_css ? 'active' : 'inactive'" type="success"
+                        <el-tag :class="!stage.page_xpath ? 'active' : 'inactive'" type="success"
                                 @click="onSelectStagePageType(stage, 'css')">CSS
                         </el-tag>
-                        <el-tag :class="!stage.page_css ? 'active' : 'inactive'" type="primary"
+                        <el-tag :class="stage.page_xpath ? 'active' : 'inactive'" type="primary"
                                 @click="onSelectStagePageType(stage, 'xpath')">XPath
                         </el-tag>
                       </el-form-item>
                       <el-form-item :label="$t('Selector')" class="page-selector">
-                        <el-input v-if="stage.page_css" v-model="stage.page_css"/>
+                        <el-input v-if="!stage.page_xpath" v-model="stage.page_css"/>
                         <el-input v-else v-model="stage.page_xpath"/>
                       </el-form-item>
                     </el-form>
                     <el-tag
-                      v-if="stage.page_css"
+                      v-if="!stage.page_xpath"
                       type="success"
                       slot="reference"
                       @click="onClickStagePage($event, stage, 'css')"
@@ -618,10 +618,14 @@ export default {
       } else if (this.isXpath) {
         newField['xpath'] = '//body'
       } else {
-        newField['css'] = 'body'
+        newField['xpath'] = '//body'
       }
       stages[newStageName] = {
         name: newStageName,
+        list_css: this.isCss ? 'body' : '',
+        list_xpath: this.isXpath ? '//body' : '',
+        page_css: '',
+        page_xpath: '',
         fields: [newField]
       }
       this.$set(this.spiderForm.config, 'stages', stages)
@@ -731,7 +735,7 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
     onCheckIsList (value, stage) {
       if (value) {
         if (!stage.list_css && !stage.list_xpath) {
-          stage.list_css = 'body'
+          stage.list_xpath = '//body'
         }
       } else {
         stage.list_css = ''
@@ -756,7 +760,7 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
     onCheckIsPage (value, stage) {
       if (value) {
         if (!stage.page_css && !stage.page_xpath) {
-          stage.page_css = 'body'
+          stage.page_xpath = '//body'
         }
       } else {
         stage.page_css = ''
diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue
index 2dbcfd81..f2b2d142 100644
--- a/frontend/src/components/TableView/FieldsTableView.vue
+++ b/frontend/src/components/TableView/FieldsTableView.vue
@@ -223,7 +223,7 @@ export default {
       }
       if (this.fields.length === 0) {
         this.fields.push({
-          css: 'body',
+          xpath: '//body',
           next_stage: ''
         })
       }
@@ -233,7 +233,7 @@ export default {
         if (row.name === this.fields[i].name) {
           this.fields.splice(i + 1, 0, {
             name: `field_${Math.floor(new Date().getTime()).toString()}`,
-            css: 'body',
+            xpath: '//body',
             next_stage: ''
           })
           break

From 3761d906c8f6c93434a81771ed39252df0929a8f Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 4 Dec 2019 14:54:49 +0800
Subject: [PATCH 45/52] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=88=AC=E8=99=AB?=
 =?UTF-8?q?=E8=A2=AB=E8=A6=86=E7=9B=96=E7=9A=84=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/routes/config_spider.go           | 10 ++++++++--
 backend/services/config_spider.go         |  7 ++++++-
 backend/services/spider_handler/spider.go |  1 -
 backend/utils/file.go                     |  1 -
 frontend/src/views/spider/SpiderList.vue  |  1 -
 5 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go
index 3e0f0e56..e387935a 100644
--- a/backend/routes/config_spider.go
+++ b/backend/routes/config_spider.go
@@ -254,8 +254,8 @@ func PostConfigSpiderConfig(c *gin.Context) {
 		return
 	}
 
-	// 根据序列化后的数据处理爬虫文件
-	if err := services.ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
+	// 校验configData
+	if err := services.ValidateSpiderfile(configData); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return
 	}
@@ -266,6 +266,12 @@ func PostConfigSpiderConfig(c *gin.Context) {
 		return
 	}
 
+	// 根据序列化后的数据处理爬虫文件
+	if err := services.ProcessSpiderFilesFromConfigData(spider, configData); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go
index adce0531..7c736cc7 100644
--- a/backend/services/config_spider.go
+++ b/backend/services/config_spider.go
@@ -245,7 +245,12 @@ func GenerateSpiderfileFromConfigData(spider model.Spider, configData entity.Con
 	}
 
 	// 打开文件
-	f, err := os.OpenFile(sfPath, os.O_WRONLY|os.O_TRUNC, 0777)
+	var f *os.File
+	if utils.Exists(sfPath) {
+		f, err = os.OpenFile(sfPath, os.O_WRONLY|os.O_TRUNC, 0777)
+	} else {
+		f, err = os.OpenFile(sfPath, os.O_CREATE, 0777)
+	}
 	if err != nil {
 		return err
 	}
diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go
index e8ec6a19..c3a2500d 100644
--- a/backend/services/spider_handler/spider.go
+++ b/backend/services/spider_handler/spider.go
@@ -100,7 +100,6 @@ func (s *SpiderSync) Download() {
 	// 创建临时文件
 	tmpFilePath := filepath.Join(tmpPath, randomId.String()+".zip")
 	tmpFile := utils.OpenFile(tmpFilePath)
-	defer utils.Close(tmpFile)
 
 	// 将该文件写入临时文件
 	if _, err := io.Copy(tmpFile, f); err != nil {
diff --git a/backend/utils/file.go b/backend/utils/file.go
index 2dacc9ed..c71b2cb0 100644
--- a/backend/utils/file.go
+++ b/backend/utils/file.go
@@ -167,7 +167,6 @@ func DeCompress(srcFile *os.File, dstPath string) error {
 			debug.PrintStack()
 			continue
 		}
-		defer Close(newFile)
 
 		// 拷贝该文件到新文件中
 		if _, err := io.Copy(newFile, srcFile); err != nil {
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index cc3d6acb..78c87a36 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -281,7 +281,6 @@ import {
 import dayjs from 'dayjs'
 import CrawlConfirmDialog from '../../components/Common/CrawlConfirmDialog'
 import StatusTag from '../../components/Status/StatusTag'
-import request from '../../api/request'
 
 export default {
   name: 'SpiderList',

From 522c3758e94823cec4e20825eb562026b950960b Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 4 Dec 2019 15:03:22 +0800
Subject: [PATCH 46/52] =?UTF-8?q?=E5=BD=93stage=E8=A2=AB=E5=88=A0=E9=99=A4?=
 =?UTF-8?q?=E6=97=B6=EF=BC=8C=E9=87=8D=E7=BD=AEnext=5Fstage=E8=A2=AB?=
 =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E4=B8=BA=E8=AF=A5stage=E7=9A=84field?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index bd31441e..ccae236a 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -607,6 +607,14 @@ export default {
       if (Object.keys(stages).length === 0) {
         this.onAddStage()
       }
+      // 重置next_stage被设置为该stage的field
+      Object.values(this.spiderForm.config.stages).forEach(_stage => {
+        _stage.fields.forEach(field => {
+          if (field.next_stage === stage.name) {
+            this.$set(field, 'next_stage', '')
+          }
+        })
+      })
     },
     onAddStage (stage) {
       const stages = JSON.parse(JSON.stringify(this.spiderForm.config.stages))

From 742b954593ed6b9b13d75d40008e7ab3bb83598e Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 4 Dec 2019 15:26:06 +0800
Subject: [PATCH 47/52] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E5=8F=AF=E8=A7=86=E5=8C=96=EF=BC=9A?=
 =?UTF-8?q?=E9=A1=BA=E5=BA=8F=E6=8E=92=E5=88=97?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 73 ++++++++++++++++---
 .../components/TableView/FieldsTableView.vue  | 13 +++-
 2 files changed, 73 insertions(+), 13 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index ccae236a..62106557 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -270,6 +270,7 @@
               type="list"
               title="List Page Fields"
               :fields="stage.fields"
+              :stage="stage"
               :stage-names="Object.keys(spiderForm.config.stages)"
             />
           </el-collapse-item>
@@ -313,9 +314,7 @@
 </template>
 
 <script>
-import {
-  mapState
-} from 'vuex'
+import {mapState} from 'vuex'
 import echarts from 'echarts'
 import FieldsTableView from '../TableView/FieldsTableView'
 import CrawlConfirmDialog from '../Common/CrawlConfirmDialog'
@@ -435,15 +434,58 @@ export default {
       return ''
     },
     stageNodes () {
-      // const elChart = document.querySelector('#process-chart')
-      // const totalWidth = Number(getComputedStyle(elChart).width.replace('px', ''))
-      const stages = Object.values(this.spiderForm.config.stages)
-      return stages.map((stage, i) => {
-        return {
-          name: stage.name,
-          ...stage
+      const startStage = this.spiderForm.config.stages[this.spiderForm.config.start_stage]
+      const nodes = []
+      const allStageNames = new Set()
+
+      let i = 0
+      let currentStage = startStage
+      while (currentStage) {
+        // 加入节点信息
+        nodes.push({
+          x: i++,
+          y: 0,
+          itemStyle: {
+            color: '#409EFF'
+          },
+          ...currentStage
+        })
+
+        // 记录该节点
+        allStageNames.add(currentStage.name)
+
+        // 设置当前阶段为下一阶段
+        currentStage = this.getNextStage(currentStage)
+
+        if (currentStage && allStageNames.has(currentStage.name)) {
+          currentStage = undefined
         }
+      }
+
+      // 加入剩余阶段
+      i = 0
+      const restStages = Object.values(this.spiderForm.config.stages)
+        .filter(stage => !allStageNames.has(stage.name))
+      restStages.forEach(stage => {
+        // 加入节点信息
+        nodes.push({
+          x: i++,
+          y: 1,
+          itemStyle: {
+            color: '#F56C6C'
+          },
+          ...stage
+        })
       })
+
+      return nodes
+      // const stages = Object.values(this.spiderForm.config.stages)
+      // return stages.map((stage, i) => {
+      //   return {
+      //     name: stage.name,
+      //     ...stage
+      //   }
+      // })
     },
     stageEdges () {
       const edges = []
@@ -647,7 +689,8 @@ export default {
           {
             animation: false,
             type: 'graph',
-            layout: 'force',
+            // layout: 'force',
+            layout: 'none',
             symbolSize: 50,
             roam: true,
             label: {
@@ -786,6 +829,14 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --
         if (!stage.page_xpath) stage.page_xpath = '//body'
         stage.page_css = ''
       }
+    },
+    getNextStageField (stage) {
+      return stage.fields.filter(f => !!f.next_stage)[0]
+    },
+    getNextStage (stage) {
+      const nextStageField = this.getNextStageField(stage)
+      if (!nextStageField) return
+      return this.spiderForm.config.stages[nextStageField.next_stage]
     }
   },
   mounted () {
diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue
index f2b2d142..87bf88f1 100644
--- a/frontend/src/components/TableView/FieldsTableView.vue
+++ b/frontend/src/components/TableView/FieldsTableView.vue
@@ -113,7 +113,7 @@
               @change="onChangeNextStage(scope.row)"
             >
               <el-option :label="$t('No Next Stage')" value=""/>
-              <el-option v-for="n in stageNames" :key="n" :label="n" :value="n"/>
+              <el-option v-for="n in filteredStageNames" :key="n" :label="n" :value="n"/>
             </el-select>
           </template>
         </el-table-column>
@@ -143,6 +143,12 @@ export default {
       type: String,
       default: ''
     },
+    stage: {
+      type: Object,
+      default () {
+        return {}
+      }
+    },
     stageNames: {
       type: Array,
       default () {
@@ -159,7 +165,10 @@ export default {
   computed: {
     ...mapState('spider', [
       'spiderForm'
-    ])
+    ]),
+    filteredStageNames () {
+      return this.stageNames.filter(n => n !== this.stage.name)
+    }
   },
   methods: {
     addField () {

From 3b686b5496cee76188580f5d4c8d34e5bd5abd3c Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 4 Dec 2019 15:26:56 +0800
Subject: [PATCH 48/52] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E5=8F=AF=E8=A7=86=E5=8C=96=EF=BC=9A?=
 =?UTF-8?q?=E9=A1=BA=E5=BA=8F=E6=8E=92=E5=88=97?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 62106557..9b1851ef 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -314,7 +314,7 @@
 </template>
 
 <script>
-import {mapState} from 'vuex'
+import { mapState } from 'vuex'
 import echarts from 'echarts'
 import FieldsTableView from '../TableView/FieldsTableView'
 import CrawlConfirmDialog from '../Common/CrawlConfirmDialog'

From 4460a91c8fca13ca9629d617036e67ff3fc392e2 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 4 Dec 2019 15:34:41 +0800
Subject: [PATCH 49/52] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8F=AF=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E5=8F=AF=E8=A7=86=E5=8C=96=EF=BC=9A?=
 =?UTF-8?q?Tooltip=E4=BF=A1=E6=81=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/src/components/Config/ConfigList.vue | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 9b1851ef..f419a36c 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -732,7 +732,18 @@ export default {
             if (!params.data.fields) return
 
             let str = ''
-            str += `<label>${this.$t('Stage')}: ${params.name}</label><br><br>`
+            str += `<label>${this.$t('Stage')}: ${params.name}</label><br>`
+            str += `<ul style="list-style: none; padding: 0; margin: 0;">`
+            // 列表
+            if (params.data.list_css || params.data.list_xpath) {
+              str += `<li><span style="display: inline-block;min-width: 50px;font-weight: bolder;text-align: right;margin-right: 3px">${this.$t('List')}: </span>${params.data.list_css || params.data.list_xpath}</li>`
+            }
+            if (params.data.page_css || params.data.page_xpath) {
+              str += `<li><span style="display: inline-block;min-width: 50px;font-weight: bolder;text-align: right;margin-right: 3px">${this.$t('Pagination')}: </span>${params.data.page_css || params.data.page_xpath}</li>`
+            }
+            str += `</ul><br>`
+
+            // 字段
             str += `<label>${this.$t('Fields')}: </label><br>`
             str += '<ul style="list-style: none; padding: 0; margin: 0;">'
             for (let i = 0; i < params.data.fields.length; i++) {

From c5fad12854ebf4edd05b072c0e48bb989a5004bd Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Wed, 4 Dec 2019 16:33:02 +0800
Subject: [PATCH 50/52] =?UTF-8?q?=E4=BC=98=E5=8C=96=20https://github.com/c?=
 =?UTF-8?q?rawlab-team/crawlab/issues/185?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../components/Common/CrawlConfirmDialog.vue  | 51 ++++++++++++++-----
 frontend/src/i18n/zh.js                       |  3 ++
 frontend/src/store/modules/spider.js          |  7 +--
 3 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue
index 2286beb2..f2ad70c2 100644
--- a/frontend/src/components/Common/CrawlConfirmDialog.vue
+++ b/frontend/src/components/Common/CrawlConfirmDialog.vue
@@ -2,13 +2,21 @@
   <el-dialog
     :title="$t('Notification')"
     :visible="visible"
+    class="crawl-confirm-dialog"
     width="480px"
     :before-close="beforeClose"
   >
     <div style="margin-bottom: 20px;">{{$t('Are you sure to run this spider?')}}</div>
-    <el-form label-width="80px">
-      <el-form-item :label="$t('Node')">
-        <el-select v-model="nodeId">
+    <el-form label-width="80px" :model="form" ref="form">
+      <el-form-item :label="$t('Run Type')" prop="runType" required inline-message>
+        <el-select v-model="form.runType" :placeholder="$t('Run Type')">
+          <el-option value="all-nodes" :label="$t('All Nodes')"/>
+          <el-option value="selected-nodes" :label="$t('Selected Nodes')"/>
+          <el-option value="random" :label="$t('Random')"/>
+        </el-select>
+      </el-form-item>
+      <el-form-item v-if="form.runType === 'selected-nodes'" prop="nodeIds" :label="$t('Node')" required inline-message>
+        <el-select v-model="form.nodeIds" :placeholder="$t('Node')" multiple clearable>
           <el-option
             v-for="op in nodeList"
             :key="op._id"
@@ -18,8 +26,8 @@
           />
         </el-select>
       </el-form-item>
-      <el-form-item :label="$t('Parameters')">
-        <el-input v-model="param" :placeholder="$t('Parameters')"></el-input>
+      <el-form-item :label="$t('Parameters')" prop="param" inline-message>
+        <el-input v-model="form.param" :placeholder="$t('Parameters')"></el-input>
       </el-form-item>
     </el-form>
     <template slot="footer">
@@ -31,6 +39,7 @@
 
 <script>
 import request from '../../api/request'
+
 export default {
   name: 'CrawlConfirmDialog',
   props: {
@@ -45,9 +54,12 @@ export default {
   },
   data () {
     return {
-      nodeId: '',
-      param: '',
-      nodeList: []
+      form: {
+        runType: 'random',
+        nodeIds: undefined,
+        param: '',
+        nodeList: []
+      }
     }
   },
   methods: {
@@ -55,12 +67,21 @@ export default {
       this.$emit('close')
     },
     onConfirm () {
-      this.$store.dispatch('spider/crawlSpider', { id: this.spiderId, nodeId: this.nodeId, param: this.param })
-        .then(() => {
-          this.$message.success(this.$t('A task has been scheduled successfully'))
+      this.$refs['form'].validate(res => {
+        if (!res) return
+
+        this.$store.dispatch('spider/crawlSpider', {
+          spiderId: this.spiderId,
+          nodeIds: this.form.nodeIds,
+          param: this.form.param,
+          runType: this.form.runType
         })
-      this.$emit('close')
-      this.$st.sendEv('爬虫', '运行')
+          .then(() => {
+            this.$message.success(this.$t('A task has been scheduled successfully'))
+          })
+        this.$emit('close')
+        this.$st.sendEv('爬虫', '运行')
+      })
     }
   },
   created () {
@@ -81,5 +102,7 @@ export default {
 </script>
 
 <style scoped>
-
+  .crawl-confirm-dialog >>> .el-form .el-form-item {
+    margin-bottom: 20px;
+  }
 </style>
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index c7d350d0..65170117 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -193,6 +193,9 @@ export default {
   'Wait Duration (sec)': '等待时长(秒)',
   'Runtime Duration (sec)': '运行时长(秒)',
   'Total Duration (sec)': '总时长(秒)',
+  'Run Type': '运行类型',
+  'Random': '随机',
+  'Selected Nodes': '指定节点',
 
   // 任务列表
   'Node': '节点',
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index 0b4c2724..f4d7b134 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -118,10 +118,11 @@ const actions = {
       })
   },
   crawlSpider ({ state, dispatch }, payload) {
-    const { id, nodeId, param } = payload
+    const { spiderId, runType, nodeIds, param } = payload
     return request.put(`/tasks`, {
-      spider_id: id,
-      node_id: nodeId,
+      spider_id: spiderId,
+      run_type: runType,
+      node_ids: nodeIds,
       param: param
     })
   },

From cf9ed1b65c0abf474bbbd2d32110f94bbd0b50e4 Mon Sep 17 00:00:00 2001
From: hantmac <hantmac@outlook.com>
Date: Wed, 4 Dec 2019 16:46:16 +0800
Subject: [PATCH 51/52] =?UTF-8?q?=E4=BF=AE=E6=94=B9isDeletePeriodically?=
 =?UTF-8?q?=E5=8F=82=E6=95=B0=E9=BB=98=E8=AE=A4=E4=B8=BAN?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/conf/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/conf/config.yml b/backend/conf/config.yml
index a5e0b23b..60d2bd41 100644
--- a/backend/conf/config.yml
+++ b/backend/conf/config.yml
@@ -15,7 +15,7 @@ redis:
 log:
   level: info
   path: "/var/logs/crawlab"
-  isDeletePeriodically: "Y"
+  isDeletePeriodically: "N"
   deleteFrequency: "@hourly"
 server:
   host: 0.0.0.0

From c230e393d738316e7f2a98c6ae6098a3d41a1852 Mon Sep 17 00:00:00 2001
From: hantmac <hantmac@outlook.com>
Date: Wed, 4 Dec 2019 16:48:36 +0800
Subject: [PATCH 52/52] fix

---
 backend/main.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/main.go b/backend/main.go
index b49efae7..92863a20 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -47,6 +47,8 @@ func main() {
 			panic(err)
 		}
 		log.Info("初始化定期清理日志配置成功")
+	}else {
+		log.Info("默认未开启定期清理日志配置")
 	}
 
 	// 初始化Mongodb数据库