prepared for configurable spiders

crawlab/constants/spider.py

@@ -1,7 +1,6 @@
 class SpiderType:
-    SCRAPY = 'scrapy'
-    PYSPIDER = 'pyspider'
-    WEBMAGIC = 'webmagic'
+    CONFIGURABLE = 'configurable'
+    CUSTOMIZED = 'customized'
 
 
 class LangType:

crawlab/routes/base.py

@@ -111,7 +111,7 @@ class BaseApi(Resource):
 
         self.after_update()
 
-        return item
+        return jsonify(item)
 
     def update(self, id: str = None) -> (dict, tuple):
         """
@@ -137,7 +137,7 @@ class BaseApi(Resource):
         # execute after_update hook
         self.after_update(id)
 
-        return item
+        return jsonify(item)
 
     def post(self, id: str = None, action: str = None):
         """

crawlab/routes/spiders.py

@@ -13,6 +13,7 @@ from werkzeug.datastructures import FileStorage
 
 from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_SOURCE_FILE_FOLDER, PROJECT_TMP_FOLDER
 from constants.node import NodeStatus
+from constants.spider import SpiderType
 from constants.task import TaskStatus
 from db.manager import db_manager
 from routes.base import BaseApi
@@ -96,6 +97,8 @@ class SpiderApi(BaseApi):
         # get a list of items
         else:
             items = []
+
+            # get customized spiders
             dirs = os.listdir(PROJECT_SOURCE_FILE_FOLDER)
             for _dir in dirs:
                 if _dir in IGNORE_DIRS:
@@ -114,6 +117,7 @@ class SpiderApi(BaseApi):
                         'src': dir_path,
                         'lang': lang,
                         'suffix_stats': stats,
+                        'type': SpiderType.CUSTOMIZED
                     })
 
                 # existing spider
@@ -123,39 +127,52 @@ class SpiderApi(BaseApi):
                     if last_deploy is not None:
                         spider['deploy_ts'] = last_deploy['finish_ts']
 
-                    # get last task
-                    last_task = db_manager.get_last_task(spider_id=spider['_id'])
-                    if last_task is not None:
-                        spider['task_ts'] = last_task['create_ts']
-
-                    # get site
-                    if spider.get('site') is not None:
-                        site = db_manager.get('sites', spider['site'])
-                        if site is not None:
-                            spider['site_name'] = site['name']
-
                     # file stats
                     stats = get_file_suffix_stats(dir_path)
 
                     # language
                     lang = get_lang_by_stats(stats)
 
+                    # spider type
+                    type_ = SpiderType.CUSTOMIZED
+
                     # update spider data
                     db_manager.update_one('spiders', id=str(spider['_id']), values={
                         'lang': lang,
+                        'type': type_,
                         'suffix_stats': stats,
                     })
 
-                    # ---------
-                    # stats
-                    # ---------
-                    # last 5-run errors
-                    spider['last_5_errors'] = get_last_n_run_errors_count(spider_id=spider['_id'], n=5)
-                    spider['last_7d_tasks'] = get_last_n_day_tasks_count(spider_id=spider['_id'], n=5)
-
                 # append spider
                 items.append(spider)
 
+            # get configurable spiders
+            for spider in db_manager.list('spiders', {'type': SpiderType.CONFIGURABLE}):
+                # append spider
+                items.append(spider)
+
+            # get other info
+            for i in range(len(items)):
+                spider = items[i]
+
+                # get site
+                if spider.get('site') is not None:
+                    site = db_manager.get('sites', spider['site'])
+                    if site is not None:
+                        items[i]['site_name'] = site['name']
+
+                # get last task
+                last_task = db_manager.get_last_task(spider_id=spider['_id'])
+                if last_task is not None:
+                    items[i]['task_ts'] = last_task['create_ts']
+
+                # ---------
+                # stats
+                # ---------
+                # last 5-run errors
+                items[i]['last_5_errors'] = get_last_n_run_errors_count(spider_id=spider['_id'], n=5)
+                items[i]['last_7d_tasks'] = get_last_n_day_tasks_count(spider_id=spider['_id'], n=5)
+
             return {
                 'status': 'ok',
                 'items': jsonify(items)

crawlab/spiders/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = spiders.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = spiders

crawlab/spiders/spiders/items.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class SpidersItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass

crawlab/spiders/spiders/middlewares.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class SpidersSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Response, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class SpidersDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

crawlab/spiders/spiders/pipelines.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+class SpidersPipeline(object):
+    def process_item(self, item, spider):
+        return item

crawlab/spiders/spiders/settings.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+
+# Scrapy settings for spiders project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://doc.scrapy.org/en/latest/topics/settings.html
+#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'spiders'
+
+SPIDER_MODULES = ['spiders.spiders']
+NEWSPIDER_MODULE = 'spiders.spiders'
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'spiders (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'spiders.middlewares.SpidersSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'spiders.middlewares.SpidersDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://doc.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    'spiders.pipelines.SpidersPipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

crawlab/spiders/spiders/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

frontend/src/components/Config/ConfigList.vue

@@ -0,0 +1,53 @@
+<template>
+  <div class="config-list">
+    <el-row>
+      <div class="button-group">
+        <el-button type="primary" @click="addEnv" icon="el-icon-plus">{{$t('Add Environment Variables')}}</el-button>
+        <el-button type="success" @click="save">{{$t('Save')}}</el-button>
+      </div>
+    </el-row>
+    <el-row>
+      <el-table :data="spiderForm.fields">
+        <el-table-column :label="$t('Field Name')">
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.name" :placeholder="$t('Variable')"></el-input>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Extract Type')">
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.type" :placeholder="$t('Value')"></el-input>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Query')">
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.query" :placeholder="$t('Value')"></el-input>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Action')">
+          <template slot-scope="scope">
+            <el-button size="mini" icon="el-icon-delete" type="danger" @click="deleteEnv(scope.$index)"></el-button>
+          </template>
+        </el-table-column>
+      </el-table>
+    </el-row>
+  </div>
+</template>
+
+<script>
+import {
+  mapState
+} from 'vuex'
+
+export default {
+  name: 'ConfigList',
+  computed: {
+    ...mapState('spider', [
+      'spiderForm'
+    ])
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

frontend/src/components/InfoView/SpiderInfoView.vue

@@ -12,10 +12,10 @@
         <el-form-item :label="$t('Spider Name')">
           <el-input v-model="spiderForm.name" :placeholder="$t('Spider Name')" :disabled="isView"></el-input>
         </el-form-item>
-        <el-form-item :label="$t('Source Folder')">
+        <el-form-item v-if="isCustomized" :label="$t('Source Folder')">
           <el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
         </el-form-item>
-        <el-form-item :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
+        <el-form-item v-if="isCustomized" :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
           <el-input v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"
                     :disabled="isView"></el-input>
         </el-form-item>
@@ -32,13 +32,12 @@
           </el-autocomplete>
         </el-form-item>
         <el-form-item :label="$t('Spider Type')">
-          <el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="isView" clearable>
-            <el-option value="scrapy" label="Scrapy"></el-option>
-            <el-option value="pyspider" label="PySpider"></el-option>
-            <el-option value="webmagic" label="WebMagic"></el-option>
+          <el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="true" clearable>
+            <el-option value="configurable" :label="$t('Configurable')"></el-option>
+            <el-option value="customized" :label="$t('Customized')"></el-option>
           </el-select>
         </el-form-item>
-        <el-form-item :label="$t('Language')">
+        <el-form-item v-if="isCustomized" :label="$t('Language')">
           <el-select v-model="spiderForm.lang" :placeholder="$t('Language')" :disabled="isView" clearable>
             <el-option value="python" label="Python"></el-option>
             <el-option value="javascript" label="JavaScript"></el-option>
@@ -50,7 +49,7 @@
     </el-row>
     <el-row class="button-container" v-if="!isView">
       <el-button v-if="isShowRun" type="danger" @click="onRun">{{$t('Run')}}</el-button>
-      <el-button type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
+      <el-button v-if="isCustomized" type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
       <el-button type="success" @click="onSave">{{$t('Save')}}</el-button>
     </el-row>
   </div>
@@ -99,13 +98,18 @@ export default {
       'spiderForm'
     ]),
     isShowRun () {
-      if (!this.spiderForm.deploy_ts) {
+      if (this.isCustomized) {
+        if (!this.spiderForm.deploy_ts) {
+          return false
+        }
+        return !!this.spiderForm.cmd
+      } else {
+        // TODO: has to add rules
         return false
       }
-      if (!this.spiderForm.cmd) {
-        return false
-      }
-      return true
+    },
+    isCustomized () {
+      return this.spiderForm.type === 'customized'
     }
   },
   methods: {

frontend/src/i18n/zh.js

@@ -13,16 +13,18 @@ export default {
   'Sites': '网站',
 
   // 标签
-  Overview: '概览',
-  Files: '文件',
+  'Overview': '概览',
+  'Files': '文件',
   'Deployed Spiders': '已部署爬虫',
   'Log': '日志',
   'Results': '结果',
   'Environment': '环境',
   'Analytics': '分析',
+  'Rules': '规则',
+  'Config': '配置',
 
   // 选择
-  Spider: '爬虫',
+  'Spider': '爬虫',
 
   // 块标题
   'Latest Tasks': '最近任务',
@@ -37,6 +39,7 @@ export default {
   REVOKED: '已取消',
 
   // 操作
+  Add: '添加',
   Run: '运行',
   Deploy: '部署',
   Save: '保存',
@@ -88,6 +91,9 @@ export default {
   'Variable': '变量',
   'Value': '值',
   'Add Environment Variables': '添加环境变量',
+  'Add Spider': '添加爬虫',
+  'Add Configurable Spider': '添加可配置爬虫',
+  'Add Customized Spider': '添加自定义爬虫',
   'Last 7-Day Tasks': '最近7天任务数',
   'Last 5-Run Errors': '最近5次运行错误数',
   '30-Day Tasks': '最近30天任务数',
@@ -98,6 +104,10 @@ export default {
   'Tasks by Node': '分节点任务数',
   'Daily Tasks': '每日任务数',
   'Daily Avg Duration (sec)': '每日平均运行时长(秒)',
+  'Configurable Spider': '可配置爬虫',
+  'Customized Spider': '自定义爬虫',
+  'Configurable': '可配置',
+  'Customized': '自定义',
 
   // 爬虫列表
   'Name': '名称',

frontend/src/store/modules/spider.js

@@ -79,13 +79,8 @@ const actions = {
   addSpider ({ state, dispatch }) {
     return request.put('/spiders', {
       name: state.spiderForm.name,
-      src: state.spiderForm.src,
-      cmd: state.spiderForm.cmd,
-      type: state.spiderForm.type,
-      lang: state.spiderForm.lang,
       col: state.spiderForm.col,
-      cron: state.spiderForm.cron,
-      cron_enabled: state.spiderForm.cron_enabled ? 1 : 0,
+      type: 'configurable',
       site: state.spiderForm.site
     })
       .then(() => {
@@ -100,8 +95,6 @@ const actions = {
       type: state.spiderForm.type,
       lang: state.spiderForm.lang,
       col: state.spiderForm.col,
-      cron: state.spiderForm.cron,
-      cron_enabled: state.spiderForm.cron_enabled ? 1 : 0,
       site: state.spiderForm.site
     })
       .then(() => {

frontend/src/views/spider/SpiderDetail.vue

@@ -13,7 +13,10 @@
       <el-tab-pane :label="$t('Overview')" name="overview">
         <spider-overview/>
       </el-tab-pane>
-      <el-tab-pane :label="$t('Files')" name="files">
+      <el-tab-pane v-if="isConfigurable" :label="$t('Config')" name="配置">
+        <config-list/>
+      </el-tab-pane>
+      <el-tab-pane v-if="isCustomized" :label="$t('Files')" name="files">
         <file-list/>
       </el-tab-pane>
       <el-tab-pane :label="$t('Environment')" name="environment">
@@ -34,10 +37,12 @@ import FileList from '../../components/FileList/FileList'
 import SpiderOverview from '../../components/Overview/SpiderOverview'
 import EnvironmentList from '../../components/Environment/EnvironmentList'
 import SpiderStats from '../../components/Stats/SpiderStats'
+import ConfigList from '../../components/Config/ConfigList'
 
 export default {
   name: 'NodeDetail',
   components: {
+    ConfigList,
     SpiderStats,
     EnvironmentList,
     FileList,
@@ -58,7 +63,13 @@ export default {
     ]),
     ...mapState('deploy', [
       'deployList'
-    ])
+    ]),
+    isCustomized () {
+      return this.spiderForm.type === 'customized'
+    },
+    isConfigurable () {
+      return this.spiderForm.type === 'configurable'
+    }
   },
   methods: {
     onTabClick () {

frontend/src/views/spider/SpiderList.vue

@@ -1,6 +1,6 @@
 <template>
   <div class="app-container">
-    <!--add popup-->
+    <!--import popup-->
     <el-dialog
       :title="$t('Import Spider')"
       :visible.sync="dialogVisible"
@@ -26,6 +26,66 @@
         <el-button v-loading="importLoading" type="primary" @click="onImport">{{$t('Import')}}</el-button>
       </span>
     </el-dialog>
+    <!--./import popup-->
+
+    <!--add dialog-->
+    <el-dialog :title="$t('Add Spider')"
+               width="40%"
+               :visible.sync="addDialogVisible"
+               :before-close="onAddDialogClose">
+      <div class="add-spider-wrapper">
+        <div @click="onAddConfigurable">
+          <el-card shadow="hover" class="add-spider-item success">
+            {{$t('Configurable Spider')}}
+          </el-card>
+        </div>
+        <div @click="onAddCustomized">
+          <el-card shadow="hover" class="add-spider-item primary">
+            {{$t('Customized Spider')}}
+          </el-card>
+        </div>
+      </div>
+    </el-dialog>
+    <!--./add dialog-->
+
+    <!--configurable spider dialog-->
+    <el-dialog :title="$t('Add Configurable Spider')"
+               width="40%"
+               :visible.sync="addConfigurableDialogVisible"
+               :before-close="onAddConfigurableDialogClose">
+      <el-form :model="spiderForm" ref="addConfigurableForm" inline-message>
+        <el-form-item :label="$t('Spider Name')" label-width="120px" prop="name" required>
+          <el-input :placeholder="$t('Spider Name')" v-model="spiderForm.name"></el-input>
+        </el-form-item>
+        <el-form-item :label="$t('Results Collection')" label-width="120px" name="col">
+          <el-input :placeholder="$t('Results Collection')" v-model="spiderForm.col"></el-input>
+        </el-form-item>
+        <el-form-item :label="$t('Site')" label-width="120px" name="site">
+          <el-autocomplete v-model="spiderForm.site"
+                           :placeholder="$t('Site')"
+                           :fetch-suggestions="fetchSiteSuggestions"
+                           @select="onAddConfigurableSiteSelect">
+          </el-autocomplete>
+        </el-form-item>
+      </el-form>
+      <span slot="footer" class="dialog-footer">
+        <el-button @click="addConfigurableDialogVisible = false">{{$t('Cancel')}}</el-button>
+        <el-button v-loading="addConfigurableLoading" type="primary"
+                   @click="onAddConfigurableSpider">{{$t('Add')}}</el-button>
+      </span>
+    </el-dialog>
+    <!--./configurable spider dialog-->
+
+    <!--customized spider dialog-->
+    <el-dialog :title="$t('Add Customized Spider')"
+               width="40%"
+               :visible.sync="addCustomizedDialogVisible"
+               :before-close="onAddCustomizedDialogClose">
+      <p>
+        {{$t('Please go to the source folder of your spiders, create a sub-folder and add your spider codes into it')}}
+      </p>
+    </el-dialog>
+    <!--./customized spider dialog-->
 
     <!--filter-->
     <div class="filter">
@@ -50,6 +110,12 @@
         <el-button type="primary" icon="fa fa-download" @click="openImportDialog">
           {{$t('Import Spiders')}}
         </el-button>
+        <el-button type="success"
+                   icon="el-icon-plus"
+                   class="btn add"
+                   @click="onAdd">
+          {{$t('Add Spider')}}
+        </el-button>
         <el-button type="success"
                    icon="el-icon-refresh"
                    class="btn refresh"
@@ -68,14 +134,11 @@
         <el-table-column v-if="col.name === 'type'"
                          :key="col.name"
                          :label="$t(col.label)"
-                         :sortable="col.sortable"
                          align="center"
                          :width="col.width">
           <template slot-scope="scope">
-            <el-tag v-if="scope.row.type === 'scrapy'">Scrapy</el-tag>
-            <el-tag type="warning" v-else-if="scope.row.type === 'pyspider'">PySpider</el-tag>
-            <el-tag type="info" v-else-if="scope.row.type === 'webmagic'">WebMagic</el-tag>
-            <el-tag type="success" v-else-if="scope.row.type">{{scope.row.type}}</el-tag>
+            <el-tag type="success" v-if="scope.row.type === 'configurable'">{{$t('Configurable')}}</el-tag>
+            <el-tag type="primary" v-else-if="scope.row.type === 'customized'">{{$t('Customized')}}</el-tag>
           </template>
         </el-table-column>
         <el-table-column v-else-if="col.name === 'lang'"
@@ -123,7 +186,7 @@
           <el-tooltip :content="$t('Remove')" placement="top">
             <el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
           </el-tooltip>
-          <el-tooltip :content="$t('Deploy')" placement="top">
+          <el-tooltip v-if="scope.row.type === 'customized'" :content="$t('Deploy')" placement="top">
             <el-button type="primary" icon="fa fa-cloud" size="mini" @click="onDeploy(scope.row)"></el-button>
           </el-tooltip>
           <el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
@@ -160,8 +223,12 @@ export default {
         pageSize: 10
       },
       importLoading: false,
+      addConfigurableLoading: false,
       isEditMode: false,
       dialogVisible: false,
+      addDialogVisible: false,
+      addConfigurableDialogVisible: false,
+      addCustomizedDialogVisible: false,
       filter: {
         keyword: ''
       },
@@ -169,7 +236,7 @@ export default {
       columns: [
         { name: 'name', label: 'Name', width: 'auto' },
         { name: 'site_name', label: 'Site', width: '120' },
-        { name: 'type', label: 'Spider Type', width: '120', sortable: true },
+        { name: 'type', label: 'Spider Type', width: '120' },
         { name: 'lang', label: 'Language', width: '120', sortable: true },
         { name: 'task_ts', label: 'Last Run', width: '160' },
         { name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' },
@@ -219,9 +286,16 @@ export default {
       console.log(value)
     },
     onAdd () {
+      this.addDialogVisible = true
+    },
+    onAddConfigurable () {
       this.$store.commit('spider/SET_SPIDER_FORM', {})
-      this.isEditMode = false
-      this.dialogVisible = true
+      this.addDialogVisible = false
+      this.addConfigurableDialogVisible = true
+    },
+    onAddCustomized () {
+      this.addDialogVisible = false
+      this.addCustomizedDialogVisible = true
     },
     onRefresh () {
       this.$store.dispatch('spider/getSpiderList')
@@ -246,10 +320,22 @@ export default {
       this.$store.commit('spider/SET_SPIDER_FORM', {})
       this.dialogVisible = false
     },
+    onAddCancel () {
+      this.addDialogVisible = false
+    },
     onDialogClose () {
       this.$store.commit('spider/SET_SPIDER_FORM', {})
       this.dialogVisible = false
     },
+    onAddDialogClose () {
+      this.addDialogVisible = false
+    },
+    onAddCustomizedDialogClose () {
+      this.addCustomizedDialogVisible = false
+    },
+    onAddConfigurableDialogClose () {
+      this.addConfigurableDialogVisible = false
+    },
     onEdit (row) {
       this.isEditMode = true
       this.$store.commit('spider/SET_SPIDER_FORM', row)
@@ -363,6 +449,21 @@ export default {
     },
     onSiteSelect (item) {
       this.$store.commit('spider/SET_FILTER_SITE', item._id)
+    },
+    onAddConfigurableSiteSelect (item) {
+      this.spiderForm.site = item._id
+    },
+    onAddConfigurableSpider () {
+      this.$refs['addConfigurableForm'].validate(res => {
+        if (res) {
+          this.addConfigurableLoading = true
+          this.$store.dispatch('spider/addSpider')
+            .finally(() => {
+              this.addConfigurableLoading = false
+              this.addConfigurableDialogVisible = false
+            })
+        }
+      })
     }
   },
   created () {
@@ -412,4 +513,37 @@ export default {
     background-color: red;
   }
 
+  .add-spider-wrapper {
+    display: flex;
+    justify-content: center;
+
+    .add-spider-item {
+      cursor: pointer;
+      width: 180px;
+      font-size: 18px;
+      height: 120px;
+      margin: 0 20px;
+      flex-basis: 40%;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+    }
+
+    .add-spider-item.primary {
+      color: #409eff;
+      background: rgba(64, 158, 255, .1);
+      border: 1px solid rgba(64, 158, 255, .1);
+    }
+
+    .add-spider-item.success {
+      color: #67c23a;
+      background: rgba(103, 194, 58, .1);
+      border: 1px solid rgba(103, 194, 58, .1);
+    }
+  }
+
+  .el-autocomplete {
+    width: 100%;
+  }
+
 </style>