enforce deploy before running a spider

crawlab/db/manager.py

@@ -147,7 +147,7 @@ class DbManager(object):
 
     def get_last_deploy(self, spider_id):
         """
-        @deprecated
+        Get latest deploy for a given spider_id
         """
         col = self.db['deploys']
         for item in col.find({'spider_id': ObjectId(spider_id)}) \
@@ -155,6 +155,16 @@ class DbManager(object):
             return item
         return None
 
+    def get_last_task(self, spider_id):
+        """
+        Get latest deploy for a given spider_id
+        """
+        col = self.db['tasks']
+        for item in col.find({'spider_id': ObjectId(spider_id)}) \
+                .sort('create_ts', DESCENDING):
+            return item
+        return None
+
     def aggregate(self, col_name: str, pipelines, **kwargs):
         """
         Perform MongoDB col.aggregate action to aggregate stats given collection name and pipelines.

crawlab/routes/spiders.py

@@ -81,7 +81,14 @@ class SpiderApi(BaseApi):
 
         # get one node
         elif id is not None:
-            return jsonify(db_manager.get('spiders', id=id))
+            spider = db_manager.get('spiders', id=id)
+
+            # get deploy
+            last_deploy = db_manager.get_last_deploy(spider_id=spider['_id'])
+            if last_deploy is not None:
+                spider['deploy_ts'] = last_deploy['finish_ts']
+
+            return jsonify(spider)
 
         # get a list of items
         else:
@@ -108,8 +115,23 @@ class SpiderApi(BaseApi):
 
                 # existing spider
                 else:
+                    # get last deploy
+                    last_deploy = db_manager.get_last_deploy(spider_id=spider['_id'])
+                    if last_deploy is not None:
+                        spider['deploy_ts'] = last_deploy['finish_ts']
+
+                    # get last task
+                    last_task = db_manager.get_last_task(spider_id=spider['_id'])
+                    if last_task is not None:
+                        spider['task_ts'] = last_task['create_ts']
+
+                    # file stats
                     stats = get_file_suffix_stats(dir_path)
+
+                    # language
                     lang = get_lang_by_stats(stats)
+
+                    # update spider data
                     db_manager.update_one('spiders', id=str(spider['_id']), values={
                         'lang': lang,
                         'suffix_stats': stats,

crawlab/routes/tasks.py

@@ -1,8 +1,9 @@
 import json
+from datetime import datetime
 
 import requests
 from bson import ObjectId
-from celery.worker.control import revoke
+from tasks.celery import celery_app
 
 from constants.task import TaskStatus
 from db.manager import db_manager
@@ -42,6 +43,8 @@ class TaskApi(BaseApi):
             task = db_manager.get(col_name=self.col_name, id=id)
             spider = db_manager.get(col_name='spiders', id=str(task['spider_id']))
             task['spider_name'] = spider['name']
+            if task.get('finish_ts') is not None:
+                task['duration'] = (task['finish_ts'] - task['create_ts']).total_seconds()
             try:
                 with open(task['log_file_path']) as f:
                     task['log'] = f.read()
@@ -61,7 +64,8 @@ class TaskApi(BaseApi):
             _spider = db_manager.get(col_name='spiders', id=str(task['spider_id']))
             if task.get('status') is None:
                 task['status'] = TaskStatus.UNAVAILABLE
-            task['spider_name'] = _spider['name']
+            if _spider:
+                task['spider_name'] = _spider['name']
             items.append(task)
         return {
             'status': 'ok',
@@ -146,11 +150,13 @@ class TaskApi(BaseApi):
     def stop(self, id):
         """
         Stop the task in progress.
-        TODO: work in progress
         :param id:
         :return:
         """
-        revoke(id, terminate=True)
+        celery_app.control.revoke(id, terminate=True)
+        db_manager.update_one('tasks', id=id, values={
+            'status': TaskStatus.REVOKED
+        })
         return {
             'id': id,
             'status': 'ok',

frontend/src/components/InfoView/SpiderInfoView.vue

@@ -61,7 +61,7 @@
       </el-form>
     </el-row>
     <el-row class="button-container" v-if="!isView">
-      <el-button type="danger" @click="onRun">{{$t('Run')}}</el-button>
+      <el-button v-if="isShowRun" type="danger" @click="onRun">{{$t('Run')}}</el-button>
       <el-button type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
       <el-button type="success" @click="onSave">{{$t('Save')}}</el-button>
     </el-row>
@@ -109,7 +109,16 @@ export default {
   computed: {
     ...mapState('spider', [
       'spiderForm'
-    ])
+    ]),
+    isShowRun () {
+      if (!this.spiderForm.deploy_ts) {
+        return false
+      }
+      if (!this.spiderForm.cmd) {
+        return false
+      }
+      return true
+    }
   },
   methods: {
     onRun () {
@@ -131,6 +140,11 @@ export default {
     },
     onDeploy () {
       const row = this.spiderForm
+
+      // save spider
+      this.$store.dispatch('spider/editSpider', row._id)
+
+      // validate fields
       this.$refs['spiderForm'].validate(res => {
         if (res) {
           this.$confirm(this.$t('Are you sure to deploy this spider?'), this.$t('Notification'), {

frontend/src/components/InfoView/TaskInfoView.vue

@@ -32,13 +32,13 @@
         </el-form-item>
         <el-form-item :label="$t('Error Message')" v-if="taskForm.status === 'FAILURE'">
           <div class="error-message">
-            {{taskForm.result}}
+            {{taskForm.log}}
           </div>
         </el-form-item>
       </el-form>
     </el-row>
     <el-row class="button-container">
-      <el-button v-if="isRunning" type="danger" @click="onStop">Stop</el-button>
+      <el-button v-if="isRunning" type="danger" @click="onStop">{{$t('Stop')}}</el-button>
       <!--<el-button type="danger" @click="onRestart">Restart</el-button>-->
     </el-row>
   </div>

frontend/src/i18n/zh.js

@@ -31,6 +31,7 @@ export default {
   SUCCESS: '成功',
   FAILURE: '错误',
   UNAVAILABLE: '未知',
+  REVOKED: '已取消',
 
   // 操作
   Run: '运行',
@@ -46,6 +47,7 @@ export default {
   Edit: '编辑',
   Remove: '删除',
   Confirm: '确认',
+  Stop: '停止',
 
   // 主页
   'Total Tasks': '总任务数',

frontend/src/store/modules/spider.js

@@ -95,6 +95,10 @@ const actions = {
       .then(response => {
         console.log(response.data)
       })
+      .then(response => {
+        dispatch('getSpiderData', id)
+        dispatch('getSpiderList')
+      })
   },
   crawlSpider ({ state, dispatch }, id) {
     return request.post(`/spiders/${id}/on_crawl`)

frontend/src/views/node/NodeList.vue

@@ -50,9 +50,9 @@
           <el-tooltip :content="$t('View')" placement="top">
             <el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
           </el-tooltip>
-          <el-tooltip :content="$t('Edit')" placement="top">
-            <el-button type="warning" icon="el-icon-edit" size="mini" @click="onView(scope.row)"></el-button>
-          </el-tooltip>
+          <!--<el-tooltip :content="$t('Edit')" placement="top">-->
+          <!--<el-button type="warning" icon="el-icon-edit" size="mini" @click="onView(scope.row)"></el-button>-->
+          <!--</el-tooltip>-->
           <el-tooltip :content="$t('Remove')" placement="top">
             <el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
           </el-tooltip>

frontend/src/views/spider/SpiderList.vue

@@ -93,21 +93,21 @@
                          :width="col.width">
         </el-table-column>
       </template>
-      <el-table-column :label="$t('Action')" align="center" width="250">
+      <el-table-column :label="$t('Action')" align="left" width="250">
         <template slot-scope="scope">
           <el-tooltip :content="$t('View')" placement="top">
             <el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
           </el-tooltip>
-          <el-tooltip :content="$t('Edit')" placement="top">
-            <el-button type="warning" icon="el-icon-edit" size="mini" @click="onView(scope.row)"></el-button>
-          </el-tooltip>
+          <!--<el-tooltip :content="$t('Edit')" placement="top">-->
+          <!--<el-button type="warning" icon="el-icon-edit" size="mini" @click="onView(scope.row)"></el-button>-->
+          <!--</el-tooltip>-->
           <el-tooltip :content="$t('Remove')" placement="top">
             <el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
           </el-tooltip>
           <el-tooltip :content="$t('Deploy')" placement="top">
             <el-button type="primary" icon="fa fa-cloud" size="mini" @click="onDeploy(scope.row)"></el-button>
           </el-tooltip>
-          <el-tooltip :content="$t('Run')" placement="top">
+          <el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
             <el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
           </el-tooltip>
         </template>
@@ -151,7 +151,7 @@ export default {
         { name: 'name', label: 'Name', width: 'auto' },
         { name: 'type', label: 'Spider Type', width: '160', sortable: true },
         { name: 'lang', label: 'Language', width: '160', sortable: true },
-        { name: 'last_run_ts', label: 'Last Run', width: '120' }
+        { name: 'task_ts', label: 'Last Run', width: '160' }
       ],
       spiderFormRules: {
         name: [{ required: true, message: 'Required Field', trigger: 'change' }]
@@ -301,6 +301,15 @@ export default {
               this.$message.success(this.$t('Deployed all spiders successfully'))
             })
         })
+    },
+    isShowRun (row) {
+      if (!row.deploy_ts) {
+        return false
+      }
+      if (!row.cmd) {
+        return false
+      }
+      return true
     }
   },
   created () {

frontend/src/views/task/TaskList.vue

@@ -71,6 +71,9 @@
           <el-tooltip :content="$t('View')" placement="top">
             <el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
           </el-tooltip>
+          <el-tooltip :content="$t('Remove')" placement="top">
+            <el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
+          </el-tooltip>
         </template>
       </el-table-column>
     </el-table>