diff --git a/.idea/httpRequests/http-requests-log.http b/.idea/httpRequests/http-requests-log.http index 54eaae7d..cb020385 100644 --- a/.idea/httpRequests/http-requests-log.http +++ b/.idea/httpRequests/http-requests-log.http @@ -1,3 +1,99 @@ +POST http://localhost:5000/api/spiders/5c63a2ddb65d151bee71d76b/crawl +Content-Type: application/json + +{} + +<> 2019-02-13T125947.200.json + +### + +POST http://localhost:5000/api/spiders/5c63a2ddb65d151bee71d76b/crawl +Content-Type: application/json + +{} + +<> 2019-02-13T125918.500.json + +### + +POST http://localhost:5000/api/spiders/5c63a2ddb65d151bee71d76b/crawl +Content-Type: application/json + +{} + +<> 2019-02-13T125714.200.json + +### + +POST http://localhost:5000/api/spiders/5c63a2ddb65d151bee71d76b +Content-Type: application/json + +{ + "spider_name": "b spider" +} + +<> 2019-02-13T125635.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "spider_name": "a spider" +} + +<> 2019-02-13T125349.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "spider_name": "a spider" +} + +<> 2019-02-13T125334.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-13T125142.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-13T125046.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "id": 999, + "value": "content" +} + +<> 2019-02-13T124900.200.json + +### + PUT http://localhost:5000/api/spiders Content-Type: application/json diff --git a/model/deploy.py b/model/deploy.py new file mode 100644 index 00000000..2d21c6fc --- /dev/null +++ b/model/deploy.py @@ -0,0 +1,10 @@ +from mongoengine import * + +from model.base import BaseModel + + +class Deploy(BaseModel): + _id = ObjectIdField() + spider_id = ObjectIdField() + version = IntField() + node_id = ObjectIdField() diff --git a/model/spider.py b/model/spider.py index 8ed698ec..0f36e686 100644 --- a/model/spider.py +++ b/model/spider.py @@ -9,4 +9,4 @@ class Spider(BaseModel): spider_type = IntField() lang_type = IntField() execute_cmd = StringField() - file_path = StringField() + src_file_path = StringField() diff --git a/model/task.py b/model/task.py index ccf5b495..9f934eb1 100644 --- a/model/task.py +++ b/model/task.py @@ -4,6 +4,6 @@ from model.base import BaseModel class Task(BaseModel): - node_id = ObjectIdField() - spider_id = ObjectIdField() + _id = ObjectIdField() + deploy_id = ObjectIdField() file_path = StringField() diff --git a/routes/base.py b/routes/base.py index d1685c60..7bcdc3cf 100644 --- a/routes/base.py +++ b/routes/base.py @@ -21,6 +21,9 @@ class BaseApi(Resource): self.parser.add_argument('page_size', type=int) self.parser.add_argument('filter', type=dict) + for arg, type in self.arguments: + self.parser.add_argument(arg, type=type) + def get(self, id=None): args = self.parser.parse_args() @@ -76,15 +79,15 @@ class BaseApi(Resource): item = db_manager.save(col_name=self.col_name, item=item) return item - def post(self, id=None): + def update(self, id=None): args = self.parser.parse_args() item = db_manager.get(col_name=self.col_name, id=id) if item is None: return { - 'status': 'ok', - 'code': 401, - 'error': 'item not exists' - } + 'status': 'ok', + 'code': 401, + 'error': 'item not exists' + }, 401 values = {} for k in args.keys(): if k not in DEFAULT_ARGS: @@ -92,5 +95,18 @@ class BaseApi(Resource): item = db_manager.update_one(col_name=self.col_name, id=id, values=values) return item + def post(self, id=None, action=None): + if action is None: + return self.update(id) + + if not hasattr(self, action): + return { + 'status': 'ok', + 'code': 400, + 'error': 'action "%s" invalid' % action + }, 400 + + return getattr(self, action)(id) + def delete(self, id=None): pass diff --git a/routes/spiders.py b/routes/spiders.py index dd12bf63..38e42c29 100644 --- a/routes/spiders.py +++ b/routes/spiders.py @@ -15,22 +15,22 @@ parser.add_argument('spider_name', type=str) class SpiderApi(BaseApi): col_name = 'spiders' + arguments = ( + ('spider_name', str), + ('spider_type', int), + ('lang_type', int), + ('execute_cmd', str), + ('src_file_path', str), + ) -class SpiderExecutorApi(Resource): - col_name = 'spiders' + def crawl(self, id): + print('crawl: %s' % id) - def post(self, id): - args = parser.parse_args() - job = execute_spider.delay(args.spider_name) - return { - 'id': job.id, - 'status': job.status, - 'spider_name': args.spider_name, - 'result': job.get(timeout=5) - } + def deploy(self, id): + print('deploy: %s' % id) -api.add_resource(SpiderExecutorApi, '/api/spiders//crawl') api.add_resource(SpiderApi, '/api/spiders', - '/api/spiders/') + '/api/spiders/', + '/api/spiders//') diff --git a/test/test.http b/test/test.http index 75595394..c5603dc8 100644 --- a/test/test.http +++ b/test/test.http @@ -5,13 +5,26 @@ # * 'ptr' and 'ptrp' create a POST request with a simple or parameter-like body; # * 'mptr' and 'fptr' create a POST request to submit a form with a text or file field (multipart/form-data); -### Send POST request with json body +### Send PUT request with json body PUT http://localhost:5000/api/spiders Content-Type: application/json { - "id": 999, - "value": "content" + "spider_name": "a spider" } +### Send POST request with json body +POST http://localhost:5000/api/spiders/5c63a2ddb65d151bee71d76b +Content-Type: application/json + +{ + "spider_name": "b spider" +} + +### Send POST request with json body by path +POST http://localhost:5000/api/spiders/5c63a2ddb65d151bee71d76b/crawl +Content-Type: application/json + +{} + ### \ No newline at end of file