diff --git a/crawlab/app.py b/crawlab/app.py index eabb2bc3..e3741dd1 100644 --- a/crawlab/app.py +++ b/crawlab/app.py @@ -103,4 +103,4 @@ if not os.path.exists(PROJECT_LOGS_FOLDER): if __name__ == '__main__': # run app instance - app.run(host=FLASK_HOST, port=FLASK_PORT, threaded=True) + app.run(host=FLASK_HOST, port=FLASK_PORT, threaded=False, processes=4) diff --git a/crawlab/routes/base.py b/crawlab/routes/base.py index e068e4a2..3bb2c1b0 100644 --- a/crawlab/routes/base.py +++ b/crawlab/routes/base.py @@ -107,11 +107,12 @@ class BaseApi(Resource): for k in args.keys(): if k not in DEFAULT_ARGS: item[k] = args.get(k) - item = db_manager.save(col_name=self.col_name, item=item) + id = db_manager.save(col_name=self.col_name, item=item) - self.after_update() + # execute after_update hook + self.after_update(id) - return jsonify(item) + return jsonify(id) def update(self, id: str = None) -> (dict, tuple): """ @@ -169,6 +170,10 @@ class BaseApi(Resource): """ # perform delete action db_manager.remove_one(col_name=self.col_name, id=id) + + # execute after_update hook + self.after_update(id) + return { 'status': 'ok', 'message': 'deleted successfully', diff --git a/crawlab/tasks/scheduler.py b/crawlab/tasks/scheduler.py index c600029c..b4aad5df 100644 --- a/crawlab/tasks/scheduler.py +++ b/crawlab/tasks/scheduler.py @@ -23,6 +23,9 @@ class Scheduler(object): scheduler = BackgroundScheduler(jobstores=jobstores) def execute_spider(self, id: str, params: str = None): + print(f'executing spider {id}') + print(f'params: {params}') + self.scheduler.print_jobs(jobstore='mongo') query = {} if params is not None: query['params'] = params @@ -33,6 +36,7 @@ class Scheduler(object): ), query) def update(self): + print('updating...') # remove all existing periodic jobs self.scheduler.remove_all_jobs() self.mongo[MONGO_DB][self.task_col].remove() @@ -57,6 +61,9 @@ class Scheduler(object): hour=hour, minute=minute, second=second) + self.scheduler.print_jobs(jobstore='mongo') + print(f'state: {self.scheduler.state}') + print(f'running: {self.scheduler.running}') def run(self): self.update() diff --git a/docker-compose.yml b/docker-compose.yml index dcf85bdb..8e4c5eaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: '3.3' services: master: - image: crawlab:latest - restart: always + image: tikazyq/crawlab:latest + container_name: crawlab volumns: - /home/yeqing/config.py:/opt/crawlab/crawlab/config/config.py # 后端配置文件 - /home/yeqing/.env.production:/opt/crawlab/frontend/.env.production # 前端配置文件 @@ -12,6 +12,10 @@ services: depends_on: - mongo - redis + entrypoint: + - /bin/sh + - /opt/crawlab/docker_init.sh + - master mongo: image: mongo:latest restart: always diff --git a/frontend/Dockerfile b/frontend/Dockerfile deleted file mode 100644 index bfcc0d49..00000000 --- a/frontend/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# images -FROM node:8.12 - -# source files -ADD . /opt/crawlab/frontend - -#更新apt-get源 使用163的源 -#RUN mv /etc/apt/sources.list /etc/apt/sources.list.bak -#COPY sources.list /etc/apt/sources.list - -# environment variables -#ENV NVM_DIR /usr/local/nvm -#ENV NODE_VERSION 8.12.0 -#ENV WORK_DIR /opt/crawlab/frontend - -# install git curl -RUN apt-get update && apt-get install -y nginx -#RUN apt-get install -y git curl - -# install nvm -#RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ -# && . $NVM_DIR/nvm.sh \ -# && nvm install v$NODE_VERSION \ -# && nvm use v$NODE_VERSION \ -# && nvm alias default v$NODE_VERSION -#ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules -#ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH - -# install frontend -RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org -RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.taobao.org - -# nginx config & start frontend -RUN cp $WORK_DIR/conf/crawlab.conf /etc/nginx/conf.d && service nginx reload - -CMD ["npm", "run", "build:prod"]