added environment variables for spider page

crawlab/routes/base.py

@@ -127,7 +127,8 @@ class BaseApi(Resource):
         values = {}
         for k in args.keys():
             if k not in DEFAULT_ARGS:
-                values[k] = args.get(k)
+                if args.get(k) is not None:
+                    values[k] = args.get(k)
         item = db_manager.update_one(col_name=self.col_name, id=id, values=values)
 
         # execute after_update hook

crawlab/routes/spiders.py

@@ -58,6 +58,9 @@ class SpiderApi(BaseApi):
 
         # spider schedule cron enabled
         ('cron_enabled', int),
+
+        # spider schedule cron enabled
+        ('envs', str),
     )
 
     def get(self, id=None, action=None):
@@ -328,6 +331,11 @@ class SpiderApi(BaseApi):
         """
         scheduler.update()
 
+    def update_envs(self, id: str):
+        args = self.parser.parse_args()
+        envs = json.loads(args.envs)
+        db_manager.update_one(col_name='spiders', id=id, values={'envs': envs})
+
 
 class SpiderImportApi(Resource):
     __doc__ = """

crawlab/tasks/spider.py

@@ -52,11 +52,22 @@ def execute_spider(self, id: str):
         'status': TaskStatus.STARTED
     })
 
-    # start the process and pass params as env variables
+    # pass params as env variables
     env = os.environ.copy()
+
+    # custom environment variables
+    if spider.get('envs'):
+        for _env in spider.get('envs'):
+            env[_env['name']] = _env['value']
+
+    # task id environment variable
     env['CRAWLAB_TASK_ID'] = task_id
+
+    # collection environment variable
     if spider.get('col'):
         env['CRAWLAB_COLLECTION'] = spider.get('col')
+
+    # start process
     p = subprocess.Popen(command.split(' '),
                          stdout=stdout.fileno(),
                          stderr=stderr.fileno(),

frontend/src/components/Environment/EnvironmentList.vue

@@ -0,0 +1,75 @@
+<template>
+  <div class="environment-list">
+    <el-row>
+      <div class="button-group">
+        <el-button type="primary" @click="addEnv" icon="el-icon-plus">{{$t('Add Environment Variables')}}</el-button>
+        <el-button type="success" @click="save">{{$t('Save')}}</el-button>
+      </div>
+    </el-row>
+    <el-row>
+      <el-table :data="spiderForm.envs">
+        <el-table-column :label="$t('Variable')">
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.name" :placeholder="$t('Variable')"></el-input>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Value')">
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.value" :placeholder="$t('Value')"></el-input>
+          </template>
+        </el-table-column>
+        <el-table-column :label="$t('Action')">
+          <template slot-scope="scope">
+            <el-button size="mini" icon="el-icon-delete" type="danger" @click="deleteEnv(scope.$index)"></el-button>
+          </template>
+        </el-table-column>
+      </el-table>
+    </el-row>
+  </div>
+</template>
+
+<script>
+import {
+  mapState
+} from 'vuex'
+
+export default {
+  name: 'EnvironmentList',
+  computed: {
+    ...mapState('spider', [
+      'spiderForm'
+    ])
+  },
+  methods: {
+    addEnv () {
+      if (!this.spiderForm.envs) {
+        this.$set(this.spiderForm, 'envs', [])
+      }
+      this.spiderForm.envs.push({
+        name: '',
+        value: ''
+      })
+      console.log(this.spiderForm)
+    },
+    deleteEnv (index) {
+      this.spiderForm.envs.splice(index, 1)
+    },
+    save () {
+      this.$store.dispatch('spider/updateSpiderEnvs')
+        .then(() => {
+          this.$message.success(this.$t('Spider info has been saved successfully'))
+        })
+        .catch(error => {
+          this.$message.error(error)
+        })
+    }
+  }
+}
+</script>
+
+<style scoped>
+  .button-group {
+    width: 100%;
+    text-align: right;
+  }
+</style>

frontend/src/i18n/zh.js

@@ -16,6 +16,7 @@ export default {
   'Deployed Spiders': '已部署爬虫',
   'Log': '日志',
   'Results': '结果',
+  'Environment': '环境',
 
   // 选择
   Spider: '爬虫',
@@ -79,6 +80,9 @@ export default {
   'Language': '语言',
   'Schedule Enabled': '是否开启定时任务',
   'Schedule Cron': '定时任务',
+  'Variable': '变量',
+  'Value': '值',
+  'Add Environment Variables': '添加环境变量',
 
   // 爬虫列表
   'Name': '名称',

frontend/src/store/modules/spider.js

@@ -5,7 +5,7 @@ const state = {
   spiderList: [],
 
   // active spider data
-  spiderForm: { _id: {} },
+  spiderForm: {},
 
   // node to deploy/run
   activeNode: {},
@@ -77,6 +77,11 @@ const actions = {
         dispatch('getSpiderList')
       })
   },
+  updateSpiderEnvs ({ state }) {
+    return request.post(`/spiders/${state.spiderForm._id}/update_envs`, {
+      envs: JSON.stringify(state.spiderForm.envs)
+    })
+  },
   getSpiderData ({ state, commit }, id) {
     return request.get(`/spiders/${id}`)
       .then(response => {

frontend/src/views/spider/SpiderDetail.vue

@@ -16,6 +16,9 @@
       <el-tab-pane :label="$t('Files')" name="files">
         <file-list/>
       </el-tab-pane>
+      <el-tab-pane :label="$t('Environment')" name="environment">
+        <environment-list/>
+      </el-tab-pane>
     </el-tabs>
   </div>
 </template>
@@ -26,10 +29,12 @@ import {
 } from 'vuex'
 import FileList from '../../components/FileList/FileList'
 import SpiderOverview from '../../components/Overview/SpiderOverview'
+import EnvironmentList from '../../components/Environment/EnvironmentList'
 
 export default {
   name: 'NodeDetail',
   components: {
+    EnvironmentList,
     FileList,
     SpiderOverview
   },

spiders/example_juejin/juejin/pipelines.py

@@ -8,9 +8,10 @@ import os
 
 from pymongo import MongoClient
 
-MONGO_HOST = '127.0.0.1'
-MONGO_PORT = 27017
-MONGO_DB = 'crawlab_test'
+MONGO_HOST = os.environ['MONGO_HOST']
+MONGO_PORT = os.environ['MONGO_PORT']
+MONGO_DB = os.environ['MONGO_DB']
+print(MONGO_HOST)
 
 
 class JuejinPipeline(object):

spiders/juejin_node/juejin_spider.js

@@ -52,8 +52,10 @@ const MongoClient = require('mongodb').MongoClient;
   });
 
   // open database connection
-  const client = await MongoClient.connect('mongodb://127.0.0.1:27017');
-  let db = await client.db('crawlab_test');
+  console.log(process.env.MONGO_HOST);
+  console.log(process.env.MONGO_PORT);
+  const client = await MongoClient.connect(`mongodb://${process.env.MONGO_HOST}:${process.env.MONGO_PORT}`);
+  let db = await client.db(process.env.MONGO_DB);
   const colName = process.env.CRAWLAB_COLLECTION || 'results_juejin';
   const taskId = process.env.CRAWLAB_TASK_ID;
   const col = db.collection(colName);