From 925f643cd8f992f580a9bde3d4c0cf61b59340aa Mon Sep 17 00:00:00 2001
From: Marvin Zhang <marvzhang@deloitte.com.cn>
Date: Wed, 29 May 2019 17:51:44 +0800
Subject: [PATCH] added extracting list fields functionality

---
 frontend/src/components/Config/ConfigList.vue | 25 ++++++++++++++++---
 frontend/src/store/modules/spider.js          |  3 +++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 5b7e3d3d..b3d4c39d 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -34,14 +34,14 @@
               </el-button>
             </el-button-group>
           </el-form-item>
-          <el-form-item :label="$t('Start URL')">
+          <el-form-item :label="$t('Start URL')" required>
             <el-input v-model="spiderForm.start_url" :placeholder="$t('Start URL')"></el-input>
           </el-form-item>
           <el-form-item :label="$t('Obey robots.txt')">
             <el-switch v-model="spiderForm.obey_robots_txt" :placeholder="$t('Obey robots.txt')"></el-switch>
           </el-form-item>
           <!--<el-form-item :label="$t('URL Pattern')">-->
-            <!--<el-input v-model="spiderForm.url_pattern" :placeholder="$t('URL Pattern')"></el-input>-->
+          <!--<el-input v-model="spiderForm.url_pattern" :placeholder="$t('URL Pattern')"></el-input>-->
           <!--</el-form-item>-->
         </el-form>
       </el-col>
@@ -82,7 +82,8 @@
     <el-row class="button-group-container">
       <div class="button-group">
         <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
-        <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">{{$t('Extract Fields')}}</el-button>
+        <el-button type="primary" @click="onExtractFields" v-loading="extractFieldsLoading">{{$t('Extract Fields')}}
+        </el-button>
         <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
         <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
       </div>
@@ -217,6 +218,24 @@ export default {
         })
     },
     onExtractFields () {
+      this.onSave()
+        .then(() => {
+          this.extractFieldsLoading = true
+          this.$store.dispatch('spider/extractFields')
+            .then(response => {
+              if (response.data.item_selector) {
+                this.$set(this.spiderForm, 'item_selector', response.data.item_selector)
+                this.$set(this.spiderForm, 'item_selector_type', 'css')
+              }
+
+              if (response.data.fields && response.data.fields.length) {
+                this.spiderForm.fields = response.data.fields
+              }
+            })
+            .finally(() => {
+              this.extractFieldsLoading = false
+            })
+        })
     }
   },
   created () {
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index 18fcce2a..831c0d0e 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -208,6 +208,9 @@ const actions = {
       .then(response => {
         commit('SET_PREVIEW_CRAWL_DATA', response.data.items)
       })
+  },
+  extractFields ({ state, commit }) {
+    return request.post(`/spiders/${state.spiderForm._id}/extract_fields`)
   }
 }