diff --git a/Jenkinsfile b/Jenkinsfile index 64d123f8..c757366e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,5 +43,14 @@ pipeline { """ } } + stage('Cleanup') { + steps { + echo 'Cleanup...' + sh """ + docker rmi `docker images | grep '' | grep -v IMAGE | awk '{ print \$3 }' | xargs` | true + docker rm `docker ps -a | grep Exited | awk '{ print \$1 }' | xargs` | true + """ + } + } } } \ No newline at end of file diff --git a/README-zh.md b/README-zh.md index 8a1cfd26..004bb608 100644 --- a/README-zh.md +++ b/README-zh.md @@ -12,7 +12,7 @@ 基于Golang的分布式爬虫管理平台，支持Python、NodeJS、Go、Java、PHP等多种编程语言以及多种爬虫框架。 -[查看演示 Demo](http://114.67.75.98:8080) | [文档](https://tikazyq.github.io/crawlab-docs) +[查看演示 Demo](http://crawlab.cn:8080) | [文档](https://tikazyq.github.io/crawlab-docs) ## 安装 diff --git a/README.md b/README.md index cc474050..6b858643 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Golang-based distributed web crawler management platform, supporting various languages including Python, NodeJS, Go, Java, PHP and various web crawler frameworks including Scrapy, Puppeteer, Selenium. -[Demo](http://114.67.75.98:8080) | [Documentation](https://tikazyq.github.io/crawlab-docs) +[Demo](http://crawlab.cn:8080) | [Documentation](https://tikazyq.github.io/crawlab-docs) ## Installation diff --git a/backend/model/spider.go b/backend/model/spider.go index 406c4bdc..c4c94edf 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -99,7 +99,7 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) { defer s.Close() // 获取爬虫列表 - var spiders []Spider + spiders := []Spider{} if err := c.Find(filter).Skip(skip).Limit(limit).All(&spiders); err != nil { debug.PrintStack() return spiders, err diff --git a/examples/worker/Dockerfile b/examples/worker/Dockerfile new file mode 100644 index 00000000..a3907914 --- /dev/null +++ b/examples/worker/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:latest + +ENV DEBIAN_FRONTEND=noninteractive + +# 添加依赖描述文件 +ADD requirements.txt /opt/crawlab/ +# 添加二进制包 +ADD crawlab /usr/local/bin/ + +RUN chmod +x /usr/local/bin/crawlab + +# 安装基本环境 +RUN apt-get update \ + && apt-get install -y curl git net-tools iputils-ping ntp python3 python3-pip \ + && apt-get clean \ + && ln -s /usr/bin/pip3 /usr/local/bin/pip \ + && ln -s /usr/bin/python3 /usr/local/bin/python + +# 安装依赖 +RUN pip install -r /opt/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple + +WORKDIR /opt/crawlab + +ENTRYPOINT ["crawlab"] \ No newline at end of file diff --git a/examples/worker/README.md b/examples/worker/README.md new file mode 100644 index 00000000..0a27310c --- /dev/null +++ b/examples/worker/README.md @@ -0,0 +1,23 @@ +# worker节点 +由于master和worker节点的存储信息是在redis上，并且使用节点所在的mac地址作为key，所以在开发本地需要启动master和worker节点会比较麻烦。 +这里是一个运行worker节点的一个例子。 + +基本思路是worker节点所需的依赖制作成一个镜像，然后把crawlab编译成二进制包，接着把配置文件和二进制包通过volumes的形式挂载到容器内部。 +这样就可以正常的运行worker节点了。之后对于容器编排的worker节点，可以直接把该镜像当成worker节点的基础镜像。 + +### 制作二进制包 +在`backend`目录下执行以下命令，生成二进制包 +``` +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o crawlab main.go +``` + + +### 构建worker镜像 +``` +docker build -t crawlab:worker . +``` + +### 运行worker节点 +``` +docker-compose up -d +``` diff --git a/examples/worker/conf/config.yml b/examples/worker/conf/config.yml new file mode 100644 index 00000000..7ea87252 --- /dev/null +++ b/examples/worker/conf/config.yml @@ -0,0 +1,25 @@ +mongo: + host: 127.0.0.1 + port: 27017 + db: crawlab_test + username: "" + password: "" +redis: + address: 127.0.0.1 + password: "" + database: 1 + port: 6379 +log: + level: info + path: "/opt/crawlab/logs" +server: + host: 0.0.0.0 + port: 8000 + master: "N" + secret: "crawlab" +spider: + path: "/opt/crawlab/spiders" +task: + workers: 4 +other: + tmppath: "/tmp" diff --git a/examples/worker/crawlab b/examples/worker/crawlab new file mode 100755 index 00000000..f4257ea8 Binary files /dev/null and b/examples/worker/crawlab differ diff --git a/examples/worker/docker-compose.yml b/examples/worker/docker-compose.yml new file mode 100644 index 00000000..3508b923 --- /dev/null +++ b/examples/worker/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3' +services: + worker: + image: crawlab:worker + container_name: crawlab-worker + volumes: + - $PWD/conf/config.yml:/opt/crawlab/conf/config.yml + - $PWD/crawlab:/usr/local/bin/crawlab \ No newline at end of file diff --git a/examples/worker/requirements.txt b/examples/worker/requirements.txt new file mode 100644 index 00000000..70cd908b --- /dev/null +++ b/examples/worker/requirements.txt @@ -0,0 +1,7 @@ +geohash2==1.1 +Scrapy==1.5.0 +APScheduler==3.5.1 +fonttools==3.34.2 +elasticsearch==5.5.3 +requests==2.22.0 +pymysql==0.9.3 \ No newline at end of file diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index e9cef838..976d98d9 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -2,7 +2,7 @@
- + {{username}} @@ -13,7 +13,7 @@ - + {{$t($store.getters['lang/lang'])}} @@ -27,6 +27,12 @@ + + + + 文档 + +
@@ -86,7 +92,6 @@ export default { .lang-list { cursor: pointer; display: inline-block; - float: right; margin-right: 35px; /*position: absolute;*/ /*right: 80px;*/ @@ -103,10 +108,21 @@ export default { cursor: pointer; height: 50px; display: inline-block; - float: right; margin-right: 35px; /*position: absolute;*/ /*right: 35px;*/ } + + .documentation { + margin-right: 35px; + + .span { + margin-left: 5px; + } + } + + .right { + float: right + } } diff --git a/jenkins/docker-compose.yml b/jenkins/docker-compose.yml index 57c85e1e..5dbabdf8 100644 --- a/jenkins/docker-compose.yml +++ b/jenkins/docker-compose.yml @@ -8,6 +8,7 @@ services: CRAWLAB_SERVER_MASTER: "Y" CRAWLAB_MONGO_HOST: "mongo" CRAWLAB_REDIS_ADDRESS: "redis" + CRAWLAB_LOG_PATH: "/var/logs/crawlab" ports: - "8080:8080" # frontend - "8000:8000" # backend