added github import

.gitignore

@@ -107,3 +107,6 @@ venv.bak/
 
 # node_modules
 node_modules/
+
+# egg-info
+*.egg-info

app.py

@@ -9,7 +9,7 @@ from config import FLASK_HOST, FLASK_PORT, PROJECT_LOGS_FOLDER
 from routes.deploys import DeployApi
 from routes.files import FileApi
 from routes.nodes import NodeApi
-from routes.spiders import SpiderApi
+from routes.spiders import SpiderApi, SpiderImportApi
 from routes.stats import StatsApi
 from routes.tasks import TaskApi
 
@@ -28,6 +28,8 @@ api.add_resource(NodeApi,
                  '/api/nodes',
                  '/api/nodes/<string:id>',
                  '/api/nodes/<string:id>/<string:action>')
+api.add_resource(SpiderImportApi,
+                 '/api/spiders/import/<string:platform>')
 api.add_resource(SpiderApi,
                  '/api/spiders',
                  '/api/spiders/<string:id>',

config.py

@@ -13,6 +13,7 @@ CELERY_MONGODB_BACKEND_SETTINGS = {
     'database': 'crawlab_test',
     'taskmeta_collection': 'tasks_celery',
 }
+CELERY_TIMEZONE = 'Asia/Shanghai'
 
 # flower variables
 FLOWER_API_ENDPOINT = 'http://localhost:5555/api'

frontend/src/store/modules/spider.js

@@ -12,7 +12,10 @@ const state = {
   activeNode: {},
 
   // upload form for importing spiders
-  importForm: {}
+  importForm: {
+    url: '',
+    type: 'github'
+  }
 }
 
 const getters = {}
@@ -114,8 +117,14 @@ const actions = {
           }).sort((a, b) => a.create_ts < b.create_ts ? 1 : -1),
           { root: true })
       })
+  },
+  importGithub ({ state }) {
+    const url = state.importForm.url
+    return request.post('/spiders/import/github', { url })
+      .then(response => {
+        console.log(response)
+      })
   }
-
 }
 
 export default {

frontend/src/views/spider/SpiderList.vue

@@ -8,22 +8,22 @@
       :before-close="onDialogClose">
       <el-form label-width="150px"
                :model="importForm"
-               ref="spiderForm"
+               ref="importForm"
                label-position="right">
         <el-form-item label="Source URL" prop="url" required>
           <el-input v-model="importForm.url" placeholder="Source URL"></el-input>
         </el-form-item>
         <el-form-item label="Source Type" prop="type" required>
           <el-select v-model="importForm.type" placeholder="Source Type">
-            <el-option value="github"></el-option>
-            <el-option value="gitlab"></el-option>
-            <el-option value="svn"></el-option>
+            <el-option value="github" label="Github"></el-option>
+            <el-option value="gitlab" label="Gitlab"></el-option>
+            <el-option value="svn" label="SVN" disabled></el-option>
           </el-select>
         </el-form-item>
       </el-form>
       <span slot="footer" class="dialog-footer">
         <el-button @click="onCancel">Cancel</el-button>
-        <el-button type="primary" @click="onImport">Import</el-button>
+        <el-button v-loading="importLoading" type="primary" @click="onImport">Import</el-button>
       </span>
     </el-dialog>
 
@@ -36,23 +36,9 @@
                 @change="onSearch">
       </el-input>
       <div class="right">
-        <el-dropdown class="btn">
-          <el-button type="primary" icon="el-icon-upload">
-            Import Spiders
-            <i class="el-icon-arrow-down el-icon--right"></i>
-          </el-button>
-          <el-dropdown-menu slot="dropdown">
-            <el-dropdown-item>
-              <div @click="openImportDialog('github')">Github</div>
-            </el-dropdown-item>
-            <el-dropdown-item disabled>
-              <span @click="openImportDialog('gitlab')">Gitlab</span>
-            </el-dropdown-item>
-            <el-dropdown-item disabled>
-              <span @click="openImportDialog('svn')">SVN</span>
-            </el-dropdown-item>
-          </el-dropdown-menu>
-        </el-dropdown>
+        <el-button type="primary" icon="el-icon-upload" @click="openImportDialog">
+          Import Spiders
+        </el-button>
         <el-button type="success"
                    icon="el-icon-refresh"
                    class="btn refresh"
@@ -158,6 +144,7 @@ export default {
         pageNum: 0,
         pageSize: 10
       },
+      importLoading: false,
       isEditMode: false,
       dialogVisible: false,
       filter: {
@@ -231,7 +218,6 @@ export default {
       this.dialogVisible = false
     },
     onEdit (row) {
-      console.log(row)
       this.isEditMode = true
       this.$store.commit('spider/SET_SPIDER_FORM', row)
       this.dialogVisible = true
@@ -268,10 +254,25 @@ export default {
       this.$store.dispatch('spider/getSpiderList')
     },
     onImport () {
-      this.dialogVisible = false
+      this.$refs.importForm.validate(valid => {
+        if (valid) {
+          this.importLoading = true
+          this.$store.dispatch('spider/importGithub')
+            .then(response => {
+              this.$message.success('Import repo sucessfully')
+              this.$store.dispatch('spider/getSpiderList')
+            })
+            .catch(response => {
+              this.$message.error(response.data.error)
+            })
+            .finally(() => {
+              this.dialogVisible = false
+              this.importLoading = false
+            })
+        }
+      })
     },
-    openImportDialog (type) {
-      this.$store.commit('spider/SET_IMPORT_FORM', { type })
+    openImportDialog () {
       this.dialogVisible = true
     }
   },

routes/spiders.py

@@ -1,13 +1,14 @@
 import json
 import os
 import shutil
+import subprocess
 from datetime import datetime
 from random import random
 
 import requests
 from bson import ObjectId
 from flask import current_app, request
-from flask_restful import reqparse
+from flask_restful import reqparse, Resource
 from werkzeug.datastructures import FileStorage
 
 from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_SOURCE_FILE_FOLDER, PROJECT_TMP_FOLDER
@@ -302,3 +303,63 @@ class SpiderApi(BaseApi):
             'status': 'ok',
             'items': items
         })
+
+
+class SpiderImportApi(Resource):
+    parser = reqparse.RequestParser()
+    arguments = [
+        ('url', str)
+    ]
+
+    def __init__(self):
+        super(SpiderImportApi).__init__()
+        for arg, type in self.arguments:
+            self.parser.add_argument(arg, type=type)
+
+    def post(self, platform=None):
+        if platform is None:
+            return {
+                       'status': 'ok',
+                       'code': 404,
+                       'error': 'platform invalid'
+                   }, 404
+
+        if not hasattr(self, platform):
+            return {
+                       'status': 'ok',
+                       'code': 400,
+                       'error': 'platform "%s" invalid' % platform
+                   }, 400
+
+        return getattr(self, platform)()
+
+    def github(self):
+        self._git()
+
+    def gitlab(self):
+        self._git()
+
+    def _git(self):
+        args = self.parser.parse_args()
+        url = args.get('url')
+        if url is None:
+            return {
+                       'status': 'ok',
+                       'code': 400,
+                       'error': 'url should not be empty'
+                   }, 400
+
+        try:
+            p = subprocess.Popen(['git', 'clone', url], cwd=PROJECT_SOURCE_FILE_FOLDER)
+            _stdout, _stderr = p.communicate()
+        except Exception as err:
+            return {
+                       'status': 'ok',
+                       'code': 500,
+                       'error': str(err)
+                   }, 500
+
+        return {
+            'status': 'ok',
+            'message': 'success'
+        }

setup.py

@@ -7,7 +7,7 @@ with open('requirements.txt') as f:
     requirements = [l for l in f.read().splitlines() if l]
 
 setup(
-    name='crawlab',
+    name='crawlab-server',
     version='0.0.1',
     url='https://github.com/tikazyq/crawlab',
     install_requires=requirements,

utils/file.py

@@ -2,7 +2,7 @@ import os
 import re
 from collections import defaultdict
 
-SUFFIX_PATTERN = r'\.(\w{,10})$'
+SUFFIX_PATTERN = r'\.([a-zA-Z]{,6})$'
 suffix_regex = re.compile(SUFFIX_PATTERN, re.IGNORECASE)
 
 SUFFIX_LANG_MAPPING = {
@@ -20,7 +20,7 @@ def get_file_suffix(file_name: str):
     if m is not None:
         return m.groups()[0]
     else:
-        return file_name
+        return None
 
 
 def get_file_list(path):
@@ -38,7 +38,8 @@ def get_file_suffix_stats(path) -> dict:
     stats = defaultdict(int)
     for file_path in get_file_list(path):
         suffix = get_file_suffix(file_path)
-        stats[suffix] += 1
+        if suffix is not None:
+            stats[suffix] += 1
     return stats