From 622ba64130cbb3768683b9124083198cbfc87bd2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 16 Feb 2019 22:56:58 +0800 Subject: [PATCH] adapt changes for SpiderList and NodeList --- .idea/httpRequests/http-requests-log.http | 15 +++++++++++++++ app.py | 10 ++++++++-- constants/node.py | 3 +++ constants/spider.py | 8 +++++++- db/manager.py | 4 ++++ model/node.py | 8 +++++--- model/spider.py | 6 +++--- routes/base.py | 4 +++- routes/{node.py => nodes.py} | 7 ++++--- routes/spiders.py | 6 +++--- test/test.http | 8 ++++++++ 11 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 constants/node.py rename routes/{node.py => nodes.py} (74%) diff --git a/.idea/httpRequests/http-requests-log.http b/.idea/httpRequests/http-requests-log.http index f2e44011..dbd2e685 100644 --- a/.idea/httpRequests/http-requests-log.http +++ b/.idea/httpRequests/http-requests-log.http @@ -9,6 +9,21 @@ Content-Type: application/json "lang_type": 1 } +<> 2019-02-16T044301.200.json + +### + +PUT http://localhost:5000/api/spiders +Content-Type: application/json + +{ + "spider_name": "baidu spider", + "cmd": "python /Users/yeqing/projects/crawlab/spiders/baidu/baidu.py", + "src": "/Users/yeqing/projects/crawlab/spiders/baidu/baidu.py", + "spider_type": 1, + "lang_type": 1 +} + <> 2019-02-13T083950.200.json ### diff --git a/app.py b/app.py index 1eccc7cc..cfd66941 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,6 @@ from celery import Celery from flask import Flask +from flask_cors import CORS from flask_restful import Api # TODO: 用配置文件启动 http://www.pythondoc.com/flask/config.html @@ -9,11 +10,16 @@ app.config['DEBUG'] = True # init flask api instance api = Api(app) +# cors support +CORS(app, supports_credentials=True) + # reference api routes -import routes.tasks +import routes.nodes import routes.spiders +import routes.deploys +import routes.tasks import routes.test # start flask app if __name__ == '__main__': - app.run() + app.run(host='0.0.0.0', port='5000') diff --git a/constants/node.py b/constants/node.py new file mode 100644 index 00000000..e4b32810 --- /dev/null +++ b/constants/node.py @@ -0,0 +1,3 @@ +class SpiderType: + OFFLINE = 0 + ONLINE = 1 diff --git a/constants/spider.py b/constants/spider.py index ef78e295..1df0e031 100644 --- a/constants/spider.py +++ b/constants/spider.py @@ -1,4 +1,10 @@ class SpiderType: SCRAPY = 1 PYSPIDER = 2 - PUPPETEER = 3 + WEBMAGIC = 3 + + +class LangType: + PYTHON = 1 + NODEJS = 2 + JAVA = 3 diff --git a/db/manager.py b/db/manager.py index c3cea2a7..b711618f 100644 --- a/db/manager.py +++ b/db/manager.py @@ -28,6 +28,10 @@ class DbManager(object): col = self.db[col_name] col.find_one_and_update({'_id': ObjectId(id)}, {'$set': values}) + def remove_one(self, col_name: str, id: str, **kwargs): + col = self.db[col_name] + col.remove({'_id': ObjectId(id)}) + def list(self, col_name: str, cond: dict, skip: int = 0, limit: int = 10, **kwargs): col = self.db[col_name] data = [] diff --git a/model/node.py b/model/node.py index adf0c3f2..4b9b4910 100644 --- a/model/node.py +++ b/model/node.py @@ -5,6 +5,8 @@ from model.base import BaseModel class Node(BaseModel): _id = ObjectIdField() - node_ip = StringField() - node_name = StringField() - node_description = StringField() + ip = StringField() + port = IntField() + name = StringField() + description = StringField() + status = IntField() diff --git a/model/spider.py b/model/spider.py index 12b810d3..eee86711 100644 --- a/model/spider.py +++ b/model/spider.py @@ -5,8 +5,8 @@ from model.base import BaseModel class Spider(BaseModel): _id = ObjectIdField() - spider_name = StringField() + name = StringField() cmd = StringField() src = StringField() - spider_type = IntField() - lang_type = IntField() + type = IntField() + lang = IntField() diff --git a/routes/base.py b/routes/base.py index 7bcdc3cf..983a2a7f 100644 --- a/routes/base.py +++ b/routes/base.py @@ -58,6 +58,8 @@ class BaseApi(Resource): skip=(page - 1) * page_size, limit=page_size) + # TODO: getting status for node + return jsonify({ 'status': 'ok', 'total_count': total_count, @@ -109,4 +111,4 @@ class BaseApi(Resource): return getattr(self, action)(id) def delete(self, id=None): - pass + db_manager.remove_one(col_name=self.col_name, id=id) diff --git a/routes/node.py b/routes/nodes.py similarity index 74% rename from routes/node.py rename to routes/nodes.py index a1db31a2..ee5a1567 100644 --- a/routes/node.py +++ b/routes/nodes.py @@ -6,9 +6,10 @@ class NodeApi(BaseApi): col_name = 'nodes' arguments = ( - ('node_ip', str), - ('node_name', str), - ('node_description', str), + ('ip', str), + ('port', int), + ('name', str), + ('description', str), ) diff --git a/routes/spiders.py b/routes/spiders.py index 0a42f01a..d0ae53c6 100644 --- a/routes/spiders.py +++ b/routes/spiders.py @@ -16,11 +16,11 @@ class SpiderApi(BaseApi): col_name = 'spiders' arguments = ( - ('spider_name', str), + ('name', str), ('cmd', str), ('src', str), - ('spider_type', int), - ('lang_type', int), + ('type', int), + ('lang', int), ) def crawl(self, id): diff --git a/test/test.http b/test/test.http index 124b3134..d15577fc 100644 --- a/test/test.http +++ b/test/test.http @@ -35,4 +35,12 @@ Content-Type: application/json {} +### + +### Send GET request with json body by path +POST http://localhost:5000/api/spiders/5c63a2ddb65d151bee71d76b/crawl +Content-Type: application/json + +{} + ### \ No newline at end of file