fixed scheduler issue

crawlab/app.py

@@ -103,4 +103,4 @@ if not os.path.exists(PROJECT_LOGS_FOLDER):
 
 if __name__ == '__main__':
     # run app instance
-    app.run(host=FLASK_HOST, port=FLASK_PORT, threaded=True)
+    app.run(host=FLASK_HOST, port=FLASK_PORT, threaded=False, processes=4)

crawlab/routes/base.py

@@ -107,11 +107,12 @@ class BaseApi(Resource):
         for k in args.keys():
             if k not in DEFAULT_ARGS:
                 item[k] = args.get(k)
-        item = db_manager.save(col_name=self.col_name, item=item)
+        id = db_manager.save(col_name=self.col_name, item=item)
 
-        self.after_update()
+        # execute after_update hook
+        self.after_update(id)
 
-        return jsonify(item)
+        return jsonify(id)
 
     def update(self, id: str = None) -> (dict, tuple):
         """
@@ -169,6 +170,10 @@ class BaseApi(Resource):
         """
         # perform delete action
         db_manager.remove_one(col_name=self.col_name, id=id)
+
+        # execute after_update hook
+        self.after_update(id)
+
         return {
             'status': 'ok',
             'message': 'deleted successfully',

crawlab/tasks/scheduler.py

@@ -23,6 +23,9 @@ class Scheduler(object):
     scheduler = BackgroundScheduler(jobstores=jobstores)
 
     def execute_spider(self, id: str, params: str = None):
+        print(f'executing spider {id}')
+        print(f'params: {params}')
+        self.scheduler.print_jobs(jobstore='mongo')
         query = {}
         if params is not None:
             query['params'] = params
@@ -33,6 +36,7 @@ class Scheduler(object):
         ), query)
 
     def update(self):
+        print('updating...')
         # remove all existing periodic jobs
         self.scheduler.remove_all_jobs()
         self.mongo[MONGO_DB][self.task_col].remove()
@@ -57,6 +61,9 @@ class Scheduler(object):
                                    hour=hour,
                                    minute=minute,
                                    second=second)
+        self.scheduler.print_jobs(jobstore='mongo')
+        print(f'state: {self.scheduler.state}')
+        print(f'running: {self.scheduler.running}')
 
     def run(self):
         self.update()

docker-compose.yml

@@ -1,8 +1,8 @@
 version: '3.3'
 services:
   master: 
-    image: crawlab:latest
+    image: tikazyq/crawlab:latest
-    restart: always
+    container_name: crawlab
     volumns:
       - /home/yeqing/config.py:/opt/crawlab/crawlab/config/config.py # 后端配置文件
       - /home/yeqing/.env.production:/opt/crawlab/frontend/.env.production # 前端配置文件
@@ -12,6 +12,10 @@ services:
     depends_on:
       - mongo
       - redis
+    entrypoint:
+      - /bin/sh
+      - /opt/crawlab/docker_init.sh
+      - master
   mongo:
     image: mongo:latest
     restart: always

frontend/Dockerfile

@@ -1,36 +0,0 @@
-# images
-FROM node:8.12
-
-# source files
-ADD . /opt/crawlab/frontend
-
-#更新apt-get源 使用163的源
-#RUN mv /etc/apt/sources.list /etc/apt/sources.list.bak
-#COPY sources.list /etc/apt/sources.list
-
-# environment variables
-#ENV NVM_DIR /usr/local/nvm
-#ENV NODE_VERSION 8.12.0
-#ENV WORK_DIR /opt/crawlab/frontend
-
-# install git curl
-RUN apt-get update && apt-get install -y nginx
-#RUN apt-get install -y git curl
-
-# install nvm
-#RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \
-#    && . $NVM_DIR/nvm.sh \
-#    && nvm install v$NODE_VERSION \
-#    && nvm use v$NODE_VERSION \
-#    && nvm alias default v$NODE_VERSION
-#ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules
-#ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
-
-# install frontend
-RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org
-RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.taobao.org
-
-# nginx config & start frontend
-RUN cp $WORK_DIR/conf/crawlab.conf /etc/nginx/conf.d && service nginx reload
-
-CMD ["npm", "run", "build:prod"]