added models

.idea/httpRequests/http-requests-log.http

@@ -0,0 +1,84 @@
+PUT http://localhost:5000/api/spiders
+Content-Type: application/json
+
+{
+  "id": 999,
+  "value": "content"
+}
+
+<> 2019-02-12T010749.200.json
+
+###
+
+PUT http://localhost:5000/api/spiders
+Content-Type: application/json
+
+{
+  "id": 999,
+  "value": "content"
+}
+
+<> 2019-02-12T010622.200.json
+
+###
+
+PUT http://localhost:5000/api/spiders
+Content-Type: application/json
+
+{
+  "id": 999,
+  "value": "content"
+}
+
+<> 2019-02-12T010615.200.json
+
+###
+
+PUT http://localhost:5000/api/spiders
+Content-Type: application/json
+
+{
+  "id": 999,
+  "value": "content"
+}
+
+<> 2019-02-12T010551.200.json
+
+###
+
+PUT http://localhost:5000/api/spiders
+Content-Type: application/json
+
+{
+  "id": 999,
+  "value": "content"
+}
+
+<> 2019-02-12T010511.500.json
+
+###
+
+PUT http://localhost:5000/api/spiders
+Content-Type: application/json
+
+{
+  "id": 999,
+  "value": "content"
+}
+
+<> 2019-02-12T010316.405.json
+
+###
+
+POST http://localhost:5000/api/spiders
+Content-Type: application/json
+
+{
+  "id": 999,
+  "value": "content"
+}
+
+<> 2019-02-12T010301.405.json
+
+###
+

constants/lang.py

@@ -0,0 +1,4 @@
+class LangType:
+    PYTHON = 1
+    NODE = 2
+    GO = 3

constants/spider.py

@@ -0,0 +1,4 @@
+class SpiderType:
+    SCRAPY = 1
+    PYSPIDER = 2
+    PUPPETEER = 3

model/base.py

@@ -0,0 +1,6 @@
+from mongoengine import *
+import datetime
+
+
+class BaseModel(Document):
+    create_ts = DateTimeField(default=datetime.datetime.utcnow)

model/node.py

@@ -0,0 +1,10 @@
+from mongoengine import *
+
+from model.base import BaseModel
+
+
+class Node(BaseModel):
+    _id = ObjectIdField()
+    node_ip = StringField()
+    node_name = StringField()
+    node_description = StringField()

model/spider.py

@@ -0,0 +1,12 @@
+from mongoengine import *
+
+from model.base import BaseModel
+
+
+class Spider(BaseModel):
+    _id = ObjectIdField()
+    spider_name = StringField()
+    spider_type = IntField()
+    lang_type = IntField()
+    execute_cmd = StringField()
+    file_path = StringField()

model/task.py

@@ -0,0 +1,9 @@
+from mongoengine import *
+
+from model.base import BaseModel
+
+
+class Task(BaseModel):
+    node_id = ObjectIdField()
+    spider_id = ObjectIdField()
+    file_path = StringField()

routes/base.py

@@ -3,6 +3,12 @@ from flask_restful import reqparse, Resource
 from db.manager import db_manager
 from utils import jsonify
 
+DEFAULT_ARGS = [
+    'page',
+    'page_size',
+    'filter'
+]
+
 
 class BaseApi(Resource):
     col_name = 'tmp'
@@ -61,8 +67,30 @@ class BaseApi(Resource):
         else:
             return jsonify(db_manager.get(col_name=self.col_name, id=id))
 
-    def update(self, id=None):
+    def put(self):
-        pass
+        args = self.parser.parse_args()
+        item = {}
+        for k in args.keys():
+            if k not in DEFAULT_ARGS:
+                item[k] = args.get(k)
+        item = db_manager.save(col_name=self.col_name, item=item)
+        return item
 
-    def remove(self, id=None):
+    def post(self, id=None):
+        args = self.parser.parse_args()
+        item = db_manager.get(col_name=self.col_name, id=id)
+        if item is None:
+            return {
+                'status': 'ok',
+                'code': 401,
+                'error': 'item not exists'
+            }
+        values = {}
+        for k in args.keys():
+            if k not in DEFAULT_ARGS:
+                values[k] = args.get(k)
+        item = db_manager.update_one(col_name=self.col_name, id=id, values=values)
+        return item
+
+    def delete(self, id=None):
         pass

routes/spiders.py

@@ -4,45 +4,20 @@ from flask_restful import reqparse, Resource
 
 from app import api
 from db.manager import db_manager
+from routes.base import BaseApi
 from tasks.spider import execute_spider
 
 # logger = get_logger('tasks')
 parser = reqparse.RequestParser()
 parser.add_argument('spider_name', type=str)
 
-# collection name
-COL_NAME = 'spiders'
 
-
+class SpiderApi(BaseApi):
-class SpiderApi(Resource):
+    col_name = 'spiders'
-    col_name = COL_NAME
-
-    def get(self, id=None):
-        args = parser.parse_args()
-        cond = {}
-        if args.filter is not None:
-            cond = json.loads(args.filter)
-        if id is None:
-            return db_manager.list(col_name=self.col_name, cond=cond, page=args.page, page_size=args.page_size)
-        else:
-            return db_manager.get(col_name=self.col_name, id=id)
-
-    def list(self):
-        args = parser.parse_args()
-        cond = {}
-        if args.filter is not None:
-            cond = json.loads(args.filter)
-        return db_manager.list(col_name=self.col_name, cond=cond, page=args.page, page_size=args.page_size)
-
-    def update(self, id=None):
-        pass
-
-    def remove(self, id=None):
-        pass
 
 
 class SpiderExecutorApi(Resource):
-    col_name = COL_NAME
+    col_name = 'spiders'
 
     def post(self, id):
         args = parser.parse_args()

test/test.http

@@ -6,10 +6,12 @@
 # * 'mptr' and 'fptr' create a POST request to submit a form with a text or file field (multipart/form-data);
 
 ### Send POST request with json body
-POST http://localhost:5000/api/test
+PUT http://localhost:5000/api/spiders
 Content-Type: application/json
 
 {
   "id": 999,
   "value": "content"
 }
+
+###