diff --git a/examples/worker/Dockerfile b/examples/worker/Dockerfile new file mode 100644 index 00000000..a3907914 --- /dev/null +++ b/examples/worker/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:latest + +ENV DEBIAN_FRONTEND=noninteractive + +# 添加依赖描述文件 +ADD requirements.txt /opt/crawlab/ +# 添加二进制包 +ADD crawlab /usr/local/bin/ + +RUN chmod +x /usr/local/bin/crawlab + +# 安装基本环境 +RUN apt-get update \ + && apt-get install -y curl git net-tools iputils-ping ntp python3 python3-pip \ + && apt-get clean \ + && ln -s /usr/bin/pip3 /usr/local/bin/pip \ + && ln -s /usr/bin/python3 /usr/local/bin/python + +# 安装依赖 +RUN pip install -r /opt/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple + +WORKDIR /opt/crawlab + +ENTRYPOINT ["crawlab"] \ No newline at end of file diff --git a/examples/worker/README.md b/examples/worker/README.md new file mode 100644 index 00000000..0a27310c --- /dev/null +++ b/examples/worker/README.md @@ -0,0 +1,23 @@ +# worker节点 +由于master和worker节点的存储信息是在redis上,并且使用节点所在的mac地址作为key,所以在开发本地需要启动master和worker节点会比较麻烦。 +这里是一个运行worker节点的一个例子。 + +基本思路是worker节点所需的依赖制作成一个镜像,然后把crawlab编译成二进制包,接着把配置文件和二进制包通过volumes的形式挂载到容器内部。 +这样就可以正常的运行worker节点了。之后对于容器编排的worker节点,可以直接把该镜像当成worker节点的基础镜像。 + +### 制作二进制包 +在`backend`目录下执行以下命令,生成二进制包 +``` +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o crawlab main.go +``` + + +### 构建worker镜像 +``` +docker build -t crawlab:worker . +``` + +### 运行worker节点 +``` +docker-compose up -d +``` diff --git a/examples/worker/conf/config.yml b/examples/worker/conf/config.yml new file mode 100644 index 00000000..7ea87252 --- /dev/null +++ b/examples/worker/conf/config.yml @@ -0,0 +1,25 @@ +mongo: + host: 127.0.0.1 + port: 27017 + db: crawlab_test + username: "" + password: "" +redis: + address: 127.0.0.1 + password: "" + database: 1 + port: 6379 +log: + level: info + path: "/opt/crawlab/logs" +server: + host: 0.0.0.0 + port: 8000 + master: "N" + secret: "crawlab" +spider: + path: "/opt/crawlab/spiders" +task: + workers: 4 +other: + tmppath: "/tmp" diff --git a/examples/worker/crawlab b/examples/worker/crawlab new file mode 100755 index 00000000..f4257ea8 Binary files /dev/null and b/examples/worker/crawlab differ diff --git a/examples/worker/docker-compose.yml b/examples/worker/docker-compose.yml new file mode 100644 index 00000000..3508b923 --- /dev/null +++ b/examples/worker/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3' +services: + worker: + image: crawlab:worker + container_name: crawlab-worker + volumes: + - $PWD/conf/config.yml:/opt/crawlab/conf/config.yml + - $PWD/crawlab:/usr/local/bin/crawlab \ No newline at end of file diff --git a/examples/worker/requirements.txt b/examples/worker/requirements.txt new file mode 100644 index 00000000..70cd908b --- /dev/null +++ b/examples/worker/requirements.txt @@ -0,0 +1,7 @@ +geohash2==1.1 +Scrapy==1.5.0 +APScheduler==3.5.1 +fonttools==3.34.2 +elasticsearch==5.5.3 +requests==2.22.0 +pymysql==0.9.3 \ No newline at end of file