git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #542 from crawlab-team/release

Browse Source 
Release
This commit is contained in:
Marvin Zhang
2020-02-11 10:11:42 +08:00
committed by GitHub
parent 3c763ef097 dde7ed5c44
commit eb3b19916d
3 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Dockerfile
View File

@@ -30,10 +30,14 @@ ENV CRAWLAB_IS_DOCKER Y
# install packages
RUN apt-get update \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget \
&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
&& ln -s /usr/bin/python3 /usr/local/bin/python
# install dumb-init
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64
RUN chmod +x /usr/local/bin/dumb-init
# install backend
RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash

6
Dockerfile.local
View File

@@ -28,10 +28,14 @@ ENV DEBIAN_FRONTEND noninteractive
# install packages
RUN chmod 777 /tmp \
&& apt-get update \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget \
&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
&& ln -s /usr/bin/python3 /usr/local/bin/python
# install dumb-init
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64
RUN chmod +x /usr/local/bin/dumb-init
# install backend
RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash -i https://pypi.tuna.tsinghua.edu.cn/simple

2
backend/template/spiders/jd_mask/Spiderfile
View File

@@ -2,4 +2,4 @@ name: "jd_mask"
display_name: "京东口罩 (Puppeteer)"
col: "results_jd"
type: "customized"
cmd: "node jd_mask_spider.js"
cmd: "dumb-init -- node jd_mask_spider.js"