diff --git a/devops/release/crawlab-master.yaml b/devops/release/crawlab-master.yaml new file mode 100644 index 00000000..e886fdd3 --- /dev/null +++ b/devops/release/crawlab-master.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Service +metadata: + name: crawlab + namespace: crawlab-release +spec: + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30098 + selector: + app: crawlab-master + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-master + namespace: crawlab-release +spec: + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-master + template: + metadata: + labels: + app: crawlab-master + spec: + containers: + - image: tikazyq/crawlab:release + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "Y" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + - name: CRAWLAB_SETTING_ALLOWREGISTER + value: "Y" + ports: + - containerPort: 8080 + name: crawlab \ No newline at end of file diff --git a/devops/release/crawlab-worker.yaml b/devops/release/crawlab-worker.yaml new file mode 100644 index 00000000..5bc2f396 --- /dev/null +++ b/devops/release/crawlab-worker.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-worker + namespace: crawlab-release +spec: + replicas: 2 + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-worker + template: + metadata: + labels: + app: crawlab-worker + spec: + containers: + - image: tikazyq/crawlab:release + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "N" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" \ No newline at end of file diff --git a/devops/release/mongo-pv.yaml b/devops/release/mongo-pv.yaml new file mode 100644 index 00000000..44503a01 --- /dev/null +++ b/devops/release/mongo-pv.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: mongo-pv-volume-release + namespace: crawlab-release + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 5Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data/crawlab-release/mongodb/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-pv-claim-release + namespace: crawlab-release +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi \ No newline at end of file diff --git a/devops/release/mongo.yaml b/devops/release/mongo.yaml new file mode 100644 index 00000000..d169174a --- /dev/null +++ b/devops/release/mongo.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongo + namespace: crawlab-release +spec: + ports: + - port: 27017 + selector: + app: mongo + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: mongo + namespace: crawlab-release +spec: + selector: + matchLabels: + app: mongo + strategy: + type: Recreate + template: + metadata: + labels: + app: mongo + spec: + containers: + - image: mongo:4 + name: mongo + ports: + - containerPort: 27017 + name: mongo + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + volumes: + - name: mongo-persistent-storage + persistentVolumeClaim: + claimName: mongo-pv-claim-release \ No newline at end of file diff --git a/devops/release/ns.yaml b/devops/release/ns.yaml new file mode 100644 index 00000000..ee77dde9 --- /dev/null +++ b/devops/release/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: crawlab-release \ No newline at end of file diff --git a/devops/release/redis.yaml b/devops/release/redis.yaml new file mode 100644 index 00000000..459a033a --- /dev/null +++ b/devops/release/redis.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: crawlab-release +spec: + ports: + - port: 6379 + selector: + app: redis + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: redis + namespace: crawlab-release +spec: + selector: + matchLabels: + app: redis + strategy: + type: Recreate + template: + metadata: + labels: + app: redis + spec: + containers: + - image: redis + name: redis + ports: + - containerPort: 6379 + name: redis \ No newline at end of file diff --git a/docker/Dockerfile.frontend.alpine b/docker/Dockerfile.frontend.alpine deleted file mode 100644 index f809ee7c..00000000 --- a/docker/Dockerfile.frontend.alpine +++ /dev/null @@ -1,27 +0,0 @@ -FROM node:8.16.0 AS frontend-build - -ADD ./frontend /app -WORKDIR /app - -# install frontend -RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org - -RUN npm run build:prod - -FROM alpine:latest - -# 安装nginx -RUN mkdir /run/nginx && apk add nginx - -# 拷贝编译文件 -COPY --from=frontend-build /app/dist /app/dist - -# 拷贝nginx代理文件 -COPY crawlab.conf /etc/nginx/conf.d - -# 拷贝执行脚本 -COPY docker_init.sh /app/docker_init.sh - -EXPOSE 8080 - -CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.master.alpine b/docker/Dockerfile.master.alpine deleted file mode 100644 index b9dbb742..00000000 --- a/docker/Dockerfile.master.alpine +++ /dev/null @@ -1,86 +0,0 @@ -FROM golang:1.12-alpine AS backend-build - -WORKDIR /go/src/app -COPY ./backend . - -ENV GO111MODULE on -ENV GOPROXY https://mirrors.aliyun.com/goproxy/ - -RUN go install -v ./... - -FROM node:lts-alpine AS frontend-build - -ARG NPM_DISABLE_SAFE_PERM=false -RUN if [ ${NPM_DISABLE_SAFE_PERM} = true ]; then \ - # run the install - echo "info: use npm unsafe-perm mode" \ - && npm config set unsafe-perm true \ -;fi - - -ADD ./frontend /app -WORKDIR /app - -# install frontend -RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org - -RUN npm run build:prod - -# images -FROM python:alpine - -ADD . /app - - -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories - -# install packages -RUN apk update && apk add --no-cache --virtual .build-deps \ - gcc \ - linux-headers \ - musl-dev \ - libffi-dev \ - libxml2-dev \ - libxslt-dev \ - openssl-dev - -# install backend -RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple - -# copy backend files -COPY --from=backend-build /go/src/app/conf ./conf -COPY --from=backend-build /go/bin/crawlab /usr/local/bin - -# install nginx -RUN apk add --no-cache nginx openrc - -RUN apk del .build-deps -# copy frontend files -COPY --from=frontend-build /app/dist /app/dist -COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d - -VOLUME [ “/sys/fs/cgroup” ] - -RUN sed -i 's/#rc_sys=""/rc_sys="lxc"/g' /etc/rc.conf && \ - echo 'rc_provide="loopback net"' >> /etc/rc.conf && \ - sed -i 's/^#\(rc_logger="YES"\)$/\1/' /etc/rc.conf && \ - sed -i '/tty/d' /etc/inittab && \ - sed -i 's/hostname $opts/# hostname $opts/g' /etc/init.d/hostname && \ - sed -i 's/mount -t tmpfs/# mount -t tmpfs/g' /lib/rc/sh/init.sh && \ - sed -i 's/cgroup_add_service /# cgroup_add_service /g' /lib/rc/sh/openrc-run.sh && \ - rm -rf /var/cache/apk/* && \ - mkdir -p /run/openrc && \ - touch /run/openrc/softlevel && \ - /sbin/openrc - -# working directory -WORKDIR /app/backend -ENV PYTHONIOENCODING utf-8 -# frontend port -EXPOSE 8080 - -# backend port -EXPOSE 8000 - -# start backend -CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.worker.alpine b/docker/Dockerfile.worker.alpine deleted file mode 100644 index 388125a2..00000000 --- a/docker/Dockerfile.worker.alpine +++ /dev/null @@ -1,43 +0,0 @@ -FROM golang:1.12-alpine AS backend-build - -WORKDIR /go/src/app -COPY ./backend . - -ENV GO111MODULE on -ENV GOPROXY https://mirrors.aliyun.com/goproxy/ - -RUN go install -v ./... - -# images -FROM python:alpine - -ADD . /app - -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories - -RUN apk update && apk add --no-cache --virtual .build-deps \ - gcc \ - linux-headers \ - musl-dev \ - libffi-dev \ - libxml2-dev \ - libxslt-dev \ - openssl-dev - -# install backend -RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple - -# copy backend files -COPY --from=backend-build /go/src/app/conf ./conf -COPY --from=backend-build /go/bin/crawlab /usr/local/bin - -RUN apk del .build-deps - -# working directory -WORKDIR /app/backend -ENV PYTHONIOENCODING utf-8 -# backend port -EXPOSE 8000 - -# start backend -CMD crawlab \ No newline at end of file diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index a42197c5..00000000 --- a/examples/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Examples - -* frontend 前端镜像制作 -* master Master节点镜像制作 -* worker Worker节点镜像制作 -* run_docker_master.sh 运行Master节点示例脚本 -* run_docker_worker.sh 运行Worker节点示例脚本 \ No newline at end of file diff --git a/examples/frontend/Dockerfile b/examples/frontend/Dockerfile deleted file mode 100644 index 9ce88b35..00000000 --- a/examples/frontend/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM node:8.16.0-alpine AS build - -ADD ./frontend /app -WORKDIR /app - -# install frontend -RUN npm install -g yarn && yarn install - -RUN npm run build:prod - -FROM alpine:latest - -# 安装nginx -RUN mkdir /run/nginx && apk add nginx - -# 拷贝编译文件 -COPY --from=build /app/dist /app/dist - -# 拷贝nginx代理文件 -COPY crawlab.conf /etc/nginx/conf.d - -# 拷贝执行脚本 -COPY docker_init.sh /app/docker_init.sh - -# 定义后端API脚本 -ENV CRAWLAB_API_ADDRESS http://localhost:8000 - -EXPOSE 8080 - -CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/examples/frontend/README.md b/examples/frontend/README.md deleted file mode 100644 index efe9493f..00000000 --- a/examples/frontend/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# 前端镜像制作 - -前端需要手动build拿到编译后的文件,然后放入此目录进行镜像构建。 - -容器运行的时候需要指定`CRAWLAB_API_ADDRESS`环境变量,用于访问后端API接口 \ No newline at end of file diff --git a/examples/frontend/crawlab.conf b/examples/frontend/crawlab.conf deleted file mode 100644 index bf8270e5..00000000 --- a/examples/frontend/crawlab.conf +++ /dev/null @@ -1,13 +0,0 @@ -server { - gzip on; - gzip_min_length 1k; - gzip_buffers 4 16k; - #gzip_http_version 1.0; - gzip_comp_level 2; - gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; - gzip_vary off; - gzip_disable "MSIE [1-6]\."; - listen 8080; - root /app/dist; - index index.html; -} diff --git a/examples/frontend/docker_init.sh b/examples/frontend/docker_init.sh deleted file mode 100755 index cfd53442..00000000 --- a/examples/frontend/docker_init.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -# replace default api path to new one -if [ "${CRAWLAB_API_ADDRESS}" = "" ]; -then - : -else - jspath=`ls /app/dist/js/app.*.js` - sed -i "s?localhost:8000?${CRAWLAB_API_ADDRESS}?g" ${jspath} -fi - -# replace base url -if [ "${CRAWLAB_BASE_URL}" = "" ]; -then - : -else - indexpath=/app/dist/index.html - sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath} - sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath} -fi - -# start nginx -nginx -g 'daemon off;' \ No newline at end of file diff --git a/examples/master/Dockerfile b/examples/master/Dockerfile deleted file mode 100644 index 7f12c733..00000000 --- a/examples/master/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM alpine:latest - -# 配置工作目录 -WORKDIR /opt/crawlab - -# 拷贝配置文件 -COPY config.yml /opt/crawlab/conf/config.yml - -# 拷贝可执行文件 -COPY crawlab /usr/local/bin - -# 创建spiders文件用于存放爬虫, 授权可执行文件 -RUN mkdir -p /opt/crawlab/spiders && chmod +x /usr/local/bin/crawlab - -# 指定为Master节点 -ENV CRAWLAB_SERVER_MASTER Y - -EXPOSE 8000 - -CMD ["crawlab"] \ No newline at end of file diff --git a/examples/master/README.md b/examples/master/README.md deleted file mode 100644 index df68e725..00000000 --- a/examples/master/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Master 节点镜像制作 - -在Dockerfile里面的二进制包,需要手动在源码目录下进行构建然后再放进来。 - -## Linux 二进制包构建 -``` -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o crawlab main.go -``` \ No newline at end of file diff --git a/examples/master/config.yml b/examples/master/config.yml deleted file mode 100644 index 3e692679..00000000 --- a/examples/master/config.yml +++ /dev/null @@ -1,32 +0,0 @@ -api: - address: "localhost:8000" -mongo: - host: "192.168.235.26" - port: 27017 - db: crawlab_local - username: "root" - password: "example" - authSource: "admin" -redis: - address: 192.168.235.0 - password: redis-1.0 - database: 29 - port: 16379 -log: - level: info - path: "/logs/crawlab" -server: - host: 0.0.0.0 - port: 8000 - master: "Y" - secret: "crawlab" - register: - # mac 或者 ip,如果是ip,则需要手动指定IP - type: "mac" - ip: "192.168.0.104" -spider: - path: "/spiders" -task: - workers: 4 -other: - tmppath: "/tmp" diff --git a/examples/run_docker_master.sh b/examples/run_docker_master.sh deleted file mode 100644 index a6c18419..00000000 --- a/examples/run_docker_master.sh +++ /dev/null @@ -1,10 +0,0 @@ -docker run -d --restart always --name crawlab \ - -e CRAWLAB_REDIS_ADDRESS=192.168.99.1:6379 \ - -e CRAWLAB_MONGO_HOST=192.168.99.1 \ - -e CRAWLAB_SERVER_MASTER=Y \ - -e CRAWLAB_API_ADDRESS=192.168.99.100:8000 \ - -e CRAWLAB_SPIDER_PATH=/app/spiders \ - -p 8080:8080 \ - -p 8000:8000 \ - -v /var/logs/crawlab:/var/logs/crawlab \ - tikazyq/crawlab:0.3.0 \ No newline at end of file diff --git a/examples/run_docker_worker.sh b/examples/run_docker_worker.sh deleted file mode 100644 index 6592c2d6..00000000 --- a/examples/run_docker_worker.sh +++ /dev/null @@ -1,6 +0,0 @@ -docker run --restart always --name crawlab \ - -e CRAWLAB_REDIS_ADDRESS=192.168.99.1:6379 \ - -e CRAWLAB_MONGO_HOST=192.168.99.1 \ - -e CRAWLAB_SERVER_MASTER=N \ - -v /var/logs/crawlab:/var/logs/crawlab \ - tikazyq/crawlab:0.3.0 \ No newline at end of file diff --git a/examples/worker/python/Dockerfile b/examples/worker/python/Dockerfile deleted file mode 100644 index a3907914..00000000 --- a/examples/worker/python/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:latest - -ENV DEBIAN_FRONTEND=noninteractive - -# 添加依赖描述文件 -ADD requirements.txt /opt/crawlab/ -# 添加二进制包 -ADD crawlab /usr/local/bin/ - -RUN chmod +x /usr/local/bin/crawlab - -# 安装基本环境 -RUN apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp python3 python3-pip \ - && apt-get clean \ - && ln -s /usr/bin/pip3 /usr/local/bin/pip \ - && ln -s /usr/bin/python3 /usr/local/bin/python - -# 安装依赖 -RUN pip install -r /opt/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - -WORKDIR /opt/crawlab - -ENTRYPOINT ["crawlab"] \ No newline at end of file diff --git a/examples/worker/python/README.md b/examples/worker/python/README.md deleted file mode 100644 index a0c2b6ed..00000000 --- a/examples/worker/python/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# 本地开发环境worker节点制作 -由于master和worker节点的存储信息是在redis上,并且使用节点所在的mac地址作为key,所以在开发本地需要启动master和worker节点会比较麻烦。 -这里是一个运行worker节点的一个例子。 - -基本思路是worker节点所需的依赖制作成一个镜像,然后把crawlab编译成二进制包,接着把配置文件和二进制包通过volumes的形式挂载到容器内部。 -这样就可以正常的运行worker节点了。之后对于容器编排的worker节点,可以直接把该镜像当成worker节点的基础镜像。 - -### 制作二进制包 -在`backend`目录下执行以下命令,生成二进制包 -``` -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o crawlab main.go -``` - - -### 构建worker镜像 -``` -docker build -t crawlab:worker . -``` - -### 运行worker节点 -``` -docker-compose up -d -``` - -如果在多台服务器使用`docker-compose.yml`进行编排,可能出现节点注册不上的问题,因为mac地址冲突了。 -可以使用`networks`定义当前节点的IP段,这样就可以正常注册到redis \ No newline at end of file diff --git a/examples/worker/python/conf/config.yml b/examples/worker/python/conf/config.yml deleted file mode 100644 index 7ea87252..00000000 --- a/examples/worker/python/conf/config.yml +++ /dev/null @@ -1,25 +0,0 @@ -mongo: - host: 127.0.0.1 - port: 27017 - db: crawlab_test - username: "" - password: "" -redis: - address: 127.0.0.1 - password: "" - database: 1 - port: 6379 -log: - level: info - path: "/opt/crawlab/logs" -server: - host: 0.0.0.0 - port: 8000 - master: "N" - secret: "crawlab" -spider: - path: "/opt/crawlab/spiders" -task: - workers: 4 -other: - tmppath: "/tmp" diff --git a/examples/worker/python/docker-compose.yml b/examples/worker/python/docker-compose.yml deleted file mode 100644 index 63d9c90c..00000000 --- a/examples/worker/python/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3' -services: - worker: - image: crawlab:worker - container_name: crawlab-worker - volumes: - - $PWD/conf/config.yml:/opt/crawlab/conf/config.yml - # 二进制包使用源码生成 - - $PWD/crawlab:/usr/local/bin/crawlab - networks: - - crawlabnet - -networks: - crawlabnet: - ipam: - driver: default - config: - - subnet: 172.30.0.0/16 \ No newline at end of file diff --git a/examples/worker/python/requirements.txt b/examples/worker/python/requirements.txt deleted file mode 100644 index 70cd908b..00000000 --- a/examples/worker/python/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -geohash2==1.1 -Scrapy==1.5.0 -APScheduler==3.5.1 -fonttools==3.34.2 -elasticsearch==5.5.3 -requests==2.22.0 -pymysql==0.9.3 \ No newline at end of file diff --git a/jenkins/develop/docker-compose.yaml b/jenkins/develop/docker-compose.yaml deleted file mode 100644 index 745c0bdc..00000000 --- a/jenkins/develop/docker-compose.yaml +++ /dev/null @@ -1,32 +0,0 @@ -version: '3.3' -services: - master: - image: "tikazyq/crawlab:develop" - environment: - CRAWLAB_API_ADDRESS: "crawlab.cn/dev/api" - CRAWLAB_BASE_URL: "/dev" - CRAWLAB_SERVER_MASTER: "Y" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - CRAWLAB_LOG_PATH: "/var/logs/crawlab" - ports: - - "8090:8080" # frontend - - "8010:8000" # backend - depends_on: - - mongo - - redis - worker: - image: "tikazyq/crawlab:develop" - environment: - CRAWLAB_SERVER_MASTER: "N" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - depends_on: - - mongo - - redis - mongo: - image: mongo:latest - restart: always - redis: - image: redis:latest - restart: always diff --git a/jenkins/master/docker-compose.yaml b/jenkins/master/docker-compose.yaml deleted file mode 100644 index 139fbb39..00000000 --- a/jenkins/master/docker-compose.yaml +++ /dev/null @@ -1,51 +0,0 @@ -version: '3.3' -services: - master: - image: "tikazyq/crawlab:master" - environment: - CRAWLAB_API_ADDRESS: "https://crawlab.cn/api" - CRAWLAB_BASE_URL: "/demo" - CRAWLAB_SERVER_MASTER: "Y" - CRAWLAB_SERVER_REGISTER_TYPE: "ip" - CRAWLAB_SERVER_REGISTER_IP: "172.19.0.1" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - CRAWLAB_LOG_PATH: "/var/logs/crawlab" - CRAWLAB_SETTING_ALLOWREGISTER: "Y" - CRAWLAB_SERVER_LANG_NODE: "Y" - ports: - - "8080:8080" # frontend - depends_on: - - mongo - - redis - volumes: - - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 - worker: - image: "tikazyq/crawlab:master" - environment: - CRAWLAB_SERVER_MASTER: "N" - CRAWLAB_SERVER_REGISTER_TYPE: "ip" - CRAWLAB_SERVER_REGISTER_IP: "172.19.0.2" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - CRAWLAB_SERVER_LANG_NODE: "Y" - depends_on: - - mongo - - redis - volumes: - - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 - mongo: - image: mongo:latest - restart: always - volumes: - - "/opt/crawlab/mongo/data/db:/data/db" - - "/opt/crawlab/mongo/tmp:/tmp" - redis: - image: redis:latest - restart: always - volumes: - - "/opt/crawlab/redis/data:/data" - splash: # use Splash to run spiders on dynamic pages - image: scrapinghub/splash - # ports: - # - "8050:8050"