From a1559339e7632dbd31cb5c37707a68109765d237 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 30 Jan 2020 11:29:37 +0800 Subject: [PATCH] updated added splash support --- Dockerfile | 2 +- Dockerfile.local | 2 +- jenkins/master/docker-compose.yaml | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 92afee28..dceaf2ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ && ln -s /usr/bin/python3 /usr/local/bin/python # install backend -RUN pip install scrapy pymongo bs4 requests crawlab-sdk +RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash # add files ADD . /app diff --git a/Dockerfile.local b/Dockerfile.local index 91e24a63..3957d154 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -33,7 +33,7 @@ RUN chmod 777 /tmp \ && ln -s /usr/bin/python3 /usr/local/bin/python # install backend -RUN pip install scrapy pymongo bs4 requests crawlab-sdk -i https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash -i https://pypi.tuna.tsinghua.edu.cn/simple # add files ADD . /app diff --git a/jenkins/master/docker-compose.yaml b/jenkins/master/docker-compose.yaml index 87d10d10..ae16079c 100644 --- a/jenkins/master/docker-compose.yaml +++ b/jenkins/master/docker-compose.yaml @@ -39,3 +39,8 @@ services: restart: always volumes: - "/opt/crawlab/redis/data:/data" + splash: # use Splash to run spiders on dynamic pages + image: scrapinghub/splash + container_name: splash + # ports: + # - "8050:8050"