From 86b547803f1f0f347bfb3e018d016cf07337911d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Thu, 15 Aug 2019 08:08:16 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=90=84=E4=B8=AA=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E7=9A=84=E9=95=9C=E5=83=8F=E5=88=B6=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/README.md | 7 ++++ examples/frontend/Dockerfile | 20 ++++++++++++ examples/frontend/README.md | 3 ++ examples/frontend/crawlab.conf | 13 ++++++++ examples/frontend/docker_init.sh | 23 +++++++++++++ examples/master/Dockerfile | 20 ++++++++++++ examples/master/README.md | 8 +++++ examples/master/config.yml | 32 +++++++++++++++++++ examples/worker/{ => python}/Dockerfile | 0 examples/worker/{ => python}/README.md | 2 +- examples/worker/{ => python}/conf/config.yml | 0 .../worker/{ => python}/docker-compose.yml | 0 examples/worker/{ => python}/requirements.txt | 0 13 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 examples/README.md create mode 100644 examples/frontend/Dockerfile create mode 100644 examples/frontend/README.md create mode 100644 examples/frontend/crawlab.conf create mode 100755 examples/frontend/docker_init.sh create mode 100644 examples/master/Dockerfile create mode 100644 examples/master/README.md create mode 100644 examples/master/config.yml rename examples/worker/{ => python}/Dockerfile (100%) rename examples/worker/{ => python}/README.md (94%) rename examples/worker/{ => python}/conf/config.yml (100%) rename examples/worker/{ => python}/docker-compose.yml (100%) rename examples/worker/{ => python}/requirements.txt (100%) diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..a42197c5 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,7 @@ +# Examples + +* frontend 前端镜像制作 +* master Master节点镜像制作 +* worker Worker节点镜像制作 +* run_docker_master.sh 运行Master节点示例脚本 +* run_docker_worker.sh 运行Worker节点示例脚本 \ No newline at end of file diff --git a/examples/frontend/Dockerfile b/examples/frontend/Dockerfile new file mode 100644 index 00000000..d313cdcb --- /dev/null +++ b/examples/frontend/Dockerfile @@ -0,0 +1,20 @@ +FROM alpine:latest + +# 安装nginx +RUN mkdir /run/nginx && apk add nginx + +# 拷贝编译文件 +COPY dist /app/dist + +# 拷贝nginx代理文件 +COPY crawlab.conf /etc/nginx/conf.d + +# 拷贝执行脚本 +COPY docker_init.sh /app/docker_init.sh + +# 定义后端API脚本 +ENV CRAWLAB_API_ADDRESS http://localhost:8000 + +EXPOSE 8080 + +CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/examples/frontend/README.md b/examples/frontend/README.md new file mode 100644 index 00000000..9f46c945 --- /dev/null +++ b/examples/frontend/README.md @@ -0,0 +1,3 @@ +# 前端镜像制作 + +前端需要手动build拿到编译后的文件，然后放入此目录进行镜像构建 \ No newline at end of file diff --git a/examples/frontend/crawlab.conf b/examples/frontend/crawlab.conf new file mode 100644 index 00000000..bf8270e5 --- /dev/null +++ b/examples/frontend/crawlab.conf @@ -0,0 +1,13 @@ +server { + gzip on; + gzip_min_length 1k; + gzip_buffers 4 16k; + #gzip_http_version 1.0; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; + gzip_vary off; + gzip_disable "MSIE [1-6]\."; + listen 8080; + root /app/dist; + index index.html; +} diff --git a/examples/frontend/docker_init.sh b/examples/frontend/docker_init.sh new file mode 100755 index 00000000..cfd53442 --- /dev/null +++ b/examples/frontend/docker_init.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# replace default api path to new one +if [ "${CRAWLAB_API_ADDRESS}" = "" ]; +then + : +else + jspath=`ls /app/dist/js/app.*.js` + sed -i "s?localhost:8000?${CRAWLAB_API_ADDRESS}?g" ${jspath} +fi + +# replace base url +if [ "${CRAWLAB_BASE_URL}" = "" ]; +then + : +else + indexpath=/app/dist/index.html + sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath} + sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath} +fi + +# start nginx +nginx -g 'daemon off;' \ No newline at end of file diff --git a/examples/master/Dockerfile b/examples/master/Dockerfile new file mode 100644 index 00000000..7f12c733 --- /dev/null +++ b/examples/master/Dockerfile @@ -0,0 +1,20 @@ +FROM alpine:latest + +# 配置工作目录 +WORKDIR /opt/crawlab + +# 拷贝配置文件 +COPY config.yml /opt/crawlab/conf/config.yml + +# 拷贝可执行文件 +COPY crawlab /usr/local/bin + +# 创建spiders文件用于存放爬虫， 授权可执行文件 +RUN mkdir -p /opt/crawlab/spiders && chmod +x /usr/local/bin/crawlab + +# 指定为Master节点 +ENV CRAWLAB_SERVER_MASTER Y + +EXPOSE 8000 + +CMD ["crawlab"] \ No newline at end of file diff --git a/examples/master/README.md b/examples/master/README.md new file mode 100644 index 00000000..df68e725 --- /dev/null +++ b/examples/master/README.md @@ -0,0 +1,8 @@ +# Master 节点镜像制作 + +在Dockerfile里面的二进制包，需要手动在源码目录下进行构建然后再放进来。 + +## Linux 二进制包构建 +``` +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o crawlab main.go +``` \ No newline at end of file diff --git a/examples/master/config.yml b/examples/master/config.yml new file mode 100644 index 00000000..3e692679 --- /dev/null +++ b/examples/master/config.yml @@ -0,0 +1,32 @@ +api: + address: "localhost:8000" +mongo: + host: "192.168.235.26" + port: 27017 + db: crawlab_local + username: "root" + password: "example" + authSource: "admin" +redis: + address: 192.168.235.0 + password: redis-1.0 + database: 29 + port: 16379 +log: + level: info + path: "/logs/crawlab" +server: + host: 0.0.0.0 + port: 8000 + master: "Y" + secret: "crawlab" + register: + # mac 或者 ip，如果是ip，则需要手动指定IP + type: "mac" + ip: "192.168.0.104" +spider: + path: "/spiders" +task: + workers: 4 +other: + tmppath: "/tmp" diff --git a/examples/worker/Dockerfile b/examples/worker/python/Dockerfile similarity index 100% rename from examples/worker/Dockerfile rename to examples/worker/python/Dockerfile diff --git a/examples/worker/README.md b/examples/worker/python/README.md similarity index 94% rename from examples/worker/README.md rename to examples/worker/python/README.md index e8638ccd..a0c2b6ed 100644 --- a/examples/worker/README.md +++ b/examples/worker/python/README.md @@ -1,4 +1,4 @@ -# worker节点 +# 本地开发环境worker节点制作 由于master和worker节点的存储信息是在redis上，并且使用节点所在的mac地址作为key，所以在开发本地需要启动master和worker节点会比较麻烦。 这里是一个运行worker节点的一个例子。 diff --git a/examples/worker/conf/config.yml b/examples/worker/python/conf/config.yml similarity index 100% rename from examples/worker/conf/config.yml rename to examples/worker/python/conf/config.yml diff --git a/examples/worker/docker-compose.yml b/examples/worker/python/docker-compose.yml similarity index 100% rename from examples/worker/docker-compose.yml rename to examples/worker/python/docker-compose.yml diff --git a/examples/worker/requirements.txt b/examples/worker/python/requirements.txt similarity index 100% rename from examples/worker/requirements.txt rename to examples/worker/python/requirements.txt