From e20c18047fcd9fa7a4650557795dfece898f6e70 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 11 Feb 2020 10:04:41 +0800 Subject: [PATCH 1/2] added dumb-init in Dockerfile --- Dockerfile | 6 +++++- Dockerfile.local | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index dceaf2ae..40757d51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,10 +30,14 @@ ENV CRAWLAB_IS_DOCKER Y # install packages RUN apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx \ + && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python +# install dumb-init +RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 +RUN chmod +x /usr/local/bin/dumb-init + # install backend RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash diff --git a/Dockerfile.local b/Dockerfile.local index eb10563d..240d84e4 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -28,10 +28,14 @@ ENV DEBIAN_FRONTEND noninteractive # install packages RUN chmod 777 /tmp \ && apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx \ + && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python +# install dumb-init +RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 +RUN chmod +x /usr/local/bin/dumb-init + # install backend RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash -i https://pypi.tuna.tsinghua.edu.cn/simple From cf7f30d7ce2ff84404d77a8e062838deffc14b85 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 11 Feb 2020 10:06:03 +0800 Subject: [PATCH 2/2] updated spiders --- backend/template/spiders/jd_mask/Spiderfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/template/spiders/jd_mask/Spiderfile b/backend/template/spiders/jd_mask/Spiderfile index b53ba963..b5134646 100644 --- a/backend/template/spiders/jd_mask/Spiderfile +++ b/backend/template/spiders/jd_mask/Spiderfile @@ -2,4 +2,4 @@ name: "jd_mask" display_name: "京东口罩 (Puppeteer)" col: "results_jd" type: "customized" -cmd: "node jd_mask_spider.js" \ No newline at end of file +cmd: "dumb-init -- node jd_mask_spider.js" \ No newline at end of file