diff --git a/.idea/httpRequests/http-requests-log.http b/.idea/httpRequests/http-requests-log.http new file mode 100644 index 00000000..54eaae7d --- /dev/null +++ b/.idea/httpRequests/http-requests-log.http @@ -0,0 +1,84 @@ +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-12T010749.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-12T010622.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-12T010615.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-12T010551.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-12T010511.500.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-12T010316.405.json + +### + +POST http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-12T010301.405.json + +### + diff --git a/constants/__init__.py b/constants/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/constants/lang.py b/constants/lang.py new file mode 100644 index 00000000..b0dfdb4e --- /dev/null +++ b/constants/lang.py @@ -0,0 +1,4 @@ +class LangType: + PYTHON = 1 + NODE = 2 + GO = 3 diff --git a/constants/spider.py b/constants/spider.py new file mode 100644 index 00000000..ef78e295 --- /dev/null +++ b/constants/spider.py @@ -0,0 +1,4 @@ +class SpiderType: + SCRAPY = 1 + PYSPIDER = 2 + PUPPETEER = 3 diff --git a/model/base.py b/model/base.py new file mode 100644 index 00000000..253cc063 --- /dev/null +++ b/model/base.py @@ -0,0 +1,6 @@ +from mongoengine import * +import datetime + + +class BaseModel(Document): + create_ts = DateTimeField(default=datetime.datetime.utcnow) diff --git a/model/node.py b/model/node.py new file mode 100644 index 00000000..adf0c3f2 --- /dev/null +++ b/model/node.py @@ -0,0 +1,10 @@ +from mongoengine import * + +from model.base import BaseModel + + +class Node(BaseModel): + _id = ObjectIdField() + node_ip = StringField() + node_name = StringField() + node_description = StringField() diff --git a/model/spider.py b/model/spider.py new file mode 100644 index 00000000..8ed698ec --- /dev/null +++ b/model/spider.py @@ -0,0 +1,12 @@ +from mongoengine import * + +from model.base import BaseModel + + +class Spider(BaseModel): + _id = ObjectIdField() + spider_name = StringField() + spider_type = IntField() + lang_type = IntField() + execute_cmd = StringField() + file_path = StringField() diff --git a/model/task.py b/model/task.py new file mode 100644 index 00000000..ccf5b495 --- /dev/null +++ b/model/task.py @@ -0,0 +1,9 @@ +from mongoengine import * + +from model.base import BaseModel + + +class Task(BaseModel): + node_id = ObjectIdField() + spider_id = ObjectIdField() + file_path = StringField() diff --git a/routes/base.py b/routes/base.py index 686467da..d1685c60 100644 --- a/routes/base.py +++ b/routes/base.py @@ -3,6 +3,12 @@ from flask_restful import reqparse, Resource from db.manager import db_manager from utils import jsonify +DEFAULT_ARGS = [ + 'page', + 'page_size', + 'filter' +] + class BaseApi(Resource): col_name = 'tmp' @@ -61,8 +67,30 @@ class BaseApi(Resource): else: return jsonify(db_manager.get(col_name=self.col_name, id=id)) - def update(self, id=None): - pass + def put(self): + args = self.parser.parse_args() + item = {} + for k in args.keys(): + if k not in DEFAULT_ARGS: + item[k] = args.get(k) + item = db_manager.save(col_name=self.col_name, item=item) + return item - def remove(self, id=None): + def post(self, id=None): + args = self.parser.parse_args() + item = db_manager.get(col_name=self.col_name, id=id) + if item is None: + return { + 'status': 'ok', + 'code': 401, + 'error': 'item not exists' + } + values = {} + for k in args.keys(): + if k not in DEFAULT_ARGS: + values[k] = args.get(k) + item = db_manager.update_one(col_name=self.col_name, id=id, values=values) + return item + + def delete(self, id=None): pass diff --git a/routes/spiders.py b/routes/spiders.py index a282c528..dd12bf63 100644 --- a/routes/spiders.py +++ b/routes/spiders.py @@ -4,45 +4,20 @@ from flask_restful import reqparse, Resource from app import api from db.manager import db_manager +from routes.base import BaseApi from tasks.spider import execute_spider # logger = get_logger('tasks') parser = reqparse.RequestParser() parser.add_argument('spider_name', type=str) -# collection name -COL_NAME = 'spiders' - -class SpiderApi(Resource): - col_name = COL_NAME - - def get(self, id=None): - args = parser.parse_args() - cond = {} - if args.filter is not None: - cond = json.loads(args.filter) - if id is None: - return db_manager.list(col_name=self.col_name, cond=cond, page=args.page, page_size=args.page_size) - else: - return db_manager.get(col_name=self.col_name, id=id) - - def list(self): - args = parser.parse_args() - cond = {} - if args.filter is not None: - cond = json.loads(args.filter) - return db_manager.list(col_name=self.col_name, cond=cond, page=args.page, page_size=args.page_size) - - def update(self, id=None): - pass - - def remove(self, id=None): - pass +class SpiderApi(BaseApi): + col_name = 'spiders' class SpiderExecutorApi(Resource): - col_name = COL_NAME + col_name = 'spiders' def post(self, id): args = parser.parse_args() diff --git a/test/test.http b/test/test.http index 81f51885..75595394 100644 --- a/test/test.http +++ b/test/test.http @@ -6,10 +6,12 @@ # * 'mptr' and 'fptr' create a POST request to submit a form with a text or file field (multipart/form-data); ### Send POST request with json body -POST http://localhost:5000/api/test +PUT http://localhost:5000/api/spiders Content-Type: application/json { "id": 999, "value": "content" -} \ No newline at end of file +} + +### \ No newline at end of file