Merge pull request #97 from wo10378931/master

添加开发环境下worker节点的运行示例

examples/worker/Dockerfile

@@ -0,0 +1,24 @@
+FROM ubuntu:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# 添加依赖描述文件
+ADD requirements.txt /opt/crawlab/
+# 添加二进制包
+ADD crawlab /usr/local/bin/
+
+RUN chmod +x /usr/local/bin/crawlab
+
+# 安装基本环境
+RUN apt-get update \
+	&& apt-get install -y curl git net-tools iputils-ping ntp python3 python3-pip \
+	&& apt-get clean \
+	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
+	&& ln -s /usr/bin/python3 /usr/local/bin/python
+
+# 安装依赖
+RUN pip install -r /opt/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
+
+WORKDIR /opt/crawlab
+
+ENTRYPOINT ["crawlab"]

examples/worker/README.md

@@ -0,0 +1,23 @@
+# worker节点
+由于master和worker节点的存储信息是在redis上，并且使用节点所在的mac地址作为key，所以在开发本地需要启动master和worker节点会比较麻烦。
+这里是一个运行worker节点的一个例子。
+
+基本思路是worker节点所需的依赖制作成一个镜像，然后把crawlab编译成二进制包，接着把配置文件和二进制包通过volumes的形式挂载到容器内部。
+这样就可以正常的运行worker节点了。之后对于容器编排的worker节点，可以直接把该镜像当成worker节点的基础镜像。
+
+### 制作二进制包
+在`backend`目录下执行以下命令，生成二进制包
+```
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o crawlab main.go
+```
+
+
+### 构建worker镜像
+```
+docker build -t crawlab:worker .
+```
+
+### 运行worker节点
+```
+docker-compose up -d
+```

examples/worker/conf/config.yml

@@ -0,0 +1,25 @@
+mongo:
+  host: 127.0.0.1
+  port: 27017
+  db: crawlab_test
+  username: ""
+  password: ""
+redis:
+  address: 127.0.0.1
+  password: ""
+  database: 1
+  port: 6379
+log:
+  level: info
+  path: "/opt/crawlab/logs"
+server:
+  host: 0.0.0.0
+  port: 8000
+  master: "N"
+  secret: "crawlab"
+spider:
+  path: "/opt/crawlab/spiders"
+task:
+  workers: 4
+other:
+  tmppath: "/tmp"

examples/worker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '3'
+services:
+  worker:
+    image: crawlab:worker
+    container_name: crawlab-worker
+    volumes:
+    - $PWD/conf/config.yml:/opt/crawlab/conf/config.yml
+    - $PWD/crawlab:/usr/local/bin/crawlab

examples/worker/requirements.txt

@@ -0,0 +1,7 @@
+geohash2==1.1
+Scrapy==1.5.0
+APScheduler==3.5.1
+fonttools==3.34.2
+elasticsearch==5.5.3
+requests==2.22.0
+pymysql==0.9.3