adapt changes for SpiderList and NodeList

.idea/httpRequests/http-requests-log.http

@@ -9,6 +9,21 @@ Content-Type: application/json
   "lang_type": 1
 }
 
+<> 2019-02-16T044301.200.json
+
+###
+
+PUT http://localhost:5000/api/spiders
+Content-Type: application/json
+
+{
+  "spider_name": "baidu spider",
+  "cmd": "python /Users/yeqing/projects/crawlab/spiders/baidu/baidu.py",
+  "src": "/Users/yeqing/projects/crawlab/spiders/baidu/baidu.py",
+  "spider_type": 1,
+  "lang_type": 1
+}
+
 <> 2019-02-13T083950.200.json
 
 ###

app.py

@@ -1,5 +1,6 @@
 from celery import Celery
 from flask import Flask
+from flask_cors import CORS
 from flask_restful import Api
 
 # TODO: 用配置文件启动 http://www.pythondoc.com/flask/config.html
@@ -9,11 +10,16 @@ app.config['DEBUG'] = True
 # init flask api instance
 api = Api(app)
 
+# cors support
+CORS(app, supports_credentials=True)
+
 # reference api routes
-import routes.tasks
+import routes.nodes
 import routes.spiders
+import routes.deploys
+import routes.tasks
 import routes.test
 
 # start flask app
 if __name__ == '__main__':
-    app.run()
+    app.run(host='0.0.0.0', port='5000')

constants/node.py

@@ -0,0 +1,3 @@
+class SpiderType:
+    OFFLINE = 0
+    ONLINE = 1

constants/spider.py

@@ -1,4 +1,10 @@
 class SpiderType:
     SCRAPY = 1
     PYSPIDER = 2
-    PUPPETEER = 3
+    WEBMAGIC = 3
+
+
+class LangType:
+    PYTHON = 1
+    NODEJS = 2
+    JAVA = 3

db/manager.py

@@ -28,6 +28,10 @@ class DbManager(object):
         col = self.db[col_name]
         col.find_one_and_update({'_id': ObjectId(id)}, {'$set': values})
 
+    def remove_one(self, col_name: str, id: str, **kwargs):
+        col = self.db[col_name]
+        col.remove({'_id': ObjectId(id)})
+
     def list(self, col_name: str, cond: dict, skip: int = 0, limit: int = 10, **kwargs):
         col = self.db[col_name]
         data = []

model/node.py

@@ -5,6 +5,8 @@ from model.base import BaseModel
 
 class Node(BaseModel):
     _id = ObjectIdField()
-    node_ip = StringField()
-    node_name = StringField()
-    node_description = StringField()
+    ip = StringField()
+    port = IntField()
+    name = StringField()
+    description = StringField()
+    status = IntField()

model/spider.py

@@ -5,8 +5,8 @@ from model.base import BaseModel
 
 class Spider(BaseModel):
     _id = ObjectIdField()
-    spider_name = StringField()
+    name = StringField()
     cmd = StringField()
     src = StringField()
-    spider_type = IntField()
-    lang_type = IntField()
+    type = IntField()
+    lang = IntField()

routes/base.py

@@ -58,6 +58,8 @@ class BaseApi(Resource):
                                     skip=(page - 1) * page_size,
                                     limit=page_size)
 
+            # TODO: getting status for node
+
             return jsonify({
                 'status': 'ok',
                 'total_count': total_count,
@@ -109,4 +111,4 @@ class BaseApi(Resource):
         return getattr(self, action)(id)
 
     def delete(self, id=None):
-        pass
+        db_manager.remove_one(col_name=self.col_name, id=id)

routes/nodes.py

@@ -6,9 +6,10 @@ class NodeApi(BaseApi):
     col_name = 'nodes'
 
     arguments = (
-        ('node_ip', str),
-        ('node_name', str),
-        ('node_description', str),
+        ('ip', str),
+        ('port', int),
+        ('name', str),
+        ('description', str),
     )
 
 

routes/spiders.py

@@ -16,11 +16,11 @@ class SpiderApi(BaseApi):
     col_name = 'spiders'
 
     arguments = (
-        ('spider_name', str),
+        ('name', str),
         ('cmd', str),
         ('src', str),
-        ('spider_type', int),
-        ('lang_type', int),
+        ('type', int),
+        ('lang', int),
     )
 
     def crawl(self, id):

test/test.http

@@ -35,4 +35,12 @@ Content-Type: application/json
 
 {}
 
+###
+
+### Send GET request with json body by path
+POST http://localhost:5000/api/spiders/5c63a2ddb65d151bee71d76b/crawl
+Content-Type: application/json
+
+{}
+
 ###