Merge pull request #1024 from crawlab-team/develop

Develop
2026-01-21 17:21:09 +01:00 · 2021-11-20 20:26:45 +08:00
parent 059b419c61 4f27473fa2
commit 5cc8891673
20 changed files with 197 additions and 200 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ tmp/
 _book/
 *.lock
 vendor/
+.crawlab
--- a/6
+++ b/6
@@ -1,4 +1,4 @@
-FROM golang:1.15 AS backend-build
+FROM golang:1.16 AS backend-build

 WORKDIR /go/src/app
 COPY ./backend .
@@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207
 # add files
 COPY ./backend/conf /app/backend/conf
 COPY ./nginx /app/nginx
-COPY ./docker_init.sh /app/docker_init.sh
+COPY ./bin /app/bin

 # copy backend files
 RUN mkdir -p /opt/bin
@@ -84,4 +84,4 @@ EXPOSE 8080
 EXPOSE 8000

 # start backend
-CMD ["/bin/bash", "/app/docker_init.sh"]
+CMD ["/bin/bash", "/app/bin/docker-init.sh"]
--- a/Dockerfile.cn
+++ b/Dockerfile.cn
@@ -1,4 +1,4 @@
-FROM golang:1.15 AS backend-build
+FROM golang:1.16 AS backend-build

 WORKDIR /go/src/app
 COPY ./backend .
@@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207
 # add files
 COPY ./backend/conf /app/backend/conf
 COPY ./nginx /app/nginx
-COPY ./docker_init.sh /app/docker_init.sh
+COPY ./bin /app/bin

 # copy backend files
 RUN mkdir -p /opt/bin
@@ -84,4 +84,4 @@ EXPOSE 8080
 EXPOSE 8000

 # start backend
-CMD ["/bin/bash", "/app/docker_init.sh"]
+CMD ["/bin/bash", "/app/bin/docker-init.sh"]
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -1,4 +1,4 @@
-FROM golang:1.15 AS backend-build
+FROM golang:1.16 AS backend-build

 WORKDIR /go/src/app
 COPY ./backend .
@@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207
 # add files
 COPY ./backend/conf /app/backend/conf
 COPY ./nginx /app/nginx
-COPY ./docker_init.sh /app/docker_init.sh
+COPY ./bin /app/bin

 # copy backend files
 RUN mkdir -p /opt/bin
@@ -84,4 +84,4 @@ EXPOSE 8080
 EXPOSE 8000

 # start backend
-CMD ["/bin/bash", "/app/docker_init.sh"]
+CMD ["/bin/bash", "/app/bin/docker-init.sh"]
--- a/backend/.air.master.conf
+++ b/backend/.air.master.conf
@@ -7,17 +7,17 @@ tmp_dir = "/tmp"

 [build]
 # Just plain old shell command. You could use `make` as well.
-cmd = "go build  -o ../tmp/main ./ "
+cmd = "go build -o ../tmp/main ./ "
 # Binary file yields from `cmd`.
 bin = "../tmp/main"
 # Customize binary.
-full_bin = "../tmp/main start"
+full_bin = "../tmp/main master"
 # Watch these filename extensions.
 include_ext = ["go", "tpl", "tmpl", "html"]
 # Ignore these filename extensions or directories.
 exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
 # Watch these directories if you specified.
-include_dir = []
+include_dir = ["../libs"]
 # Exclude files.
 exclude_file = []
 # This log file places in your tmp_dir.
--- a/backend/.air.worker.conf
+++ b/backend/.air.worker.conf
@@ -0,0 +1,47 @@
+# Config file for [Air](https://github.com/cosmtrek/air) in TOML format
+
+# Working directory
+# . or absolute path, please note that the directories following must be under root.
+root = "."
+tmp_dir = "/tmp"
+
+[build]
+# Just plain old shell command. You could use `make` as well.
+cmd = "go build -o ../tmp/main ./ "
+# Binary file yields from `cmd`.
+bin = "../tmp/main"
+# Customize binary.
+full_bin = "../tmp/main worker"
+# Watch these filename extensions.
+include_ext = ["go", "tpl", "tmpl", "html"]
+# Ignore these filename extensions or directories.
+exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
+# Watch these directories if you specified.
+include_dir = ["../libs"]
+# Exclude files.
+exclude_file = []
+# This log file places in your tmp_dir.
+log = "air.log"
+# It's not necessary to trigger build each time file changes if it's too frequent.
+delay = 1000 # ms
+# Stop running old binary when build errors occur.
+stop_on_error = true
+# Send Interrupt signal before killing process (windows does not support this feature)
+send_interrupt = false
+# Delay after sending Interrupt signal
+kill_delay = 500 # ms
+
+[log]
+# Show log time
+time = false
+
+[color]
+# Customize each part's color. If no color found, use the raw app log.
+main = "magenta"
+watcher = "cyan"
+build = "yellow"
+runner = "green"
+
+[misc]
+# Delete tmp directory on exit
+clean_on_exit = true
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -1,10 +1,10 @@
 module crawlab

-go 1.15
+go 1.16

 require (
 	github.com/apex/log v1.9.0
-	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211120.1848
 	github.com/crawlab-team/go-trace v0.1.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/spf13/cobra v1.1.3
--- a/backend/go.mod.local
+++ b/backend/go.mod.local
@@ -0,0 +1,21 @@
+module crawlab
+
+go 1.15
+
+replace (
+    github.com/crawlab-team/crawlab-core => /libs/crawlab-team/crawlab-core
+    github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs
+    github.com/crawlab-team/crawlab-fs => /libs/crawlab-team/crawlab-fs
+    github.com/crawlab-team/crawlab-db => /libs/crawlab-team/crawlab-db
+)
+
+require (
+	github.com/apex/log v1.9.0
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050
+	github.com/crawlab-team/go-trace v0.1.0
+	github.com/gin-gonic/gin v1.6.3
+	github.com/spf13/cobra v1.1.3
+	github.com/spf13/viper v1.7.1
+	go.mongodb.org/mongo-driver v1.6.0 // indirect
+	go.uber.org/dig v1.10.0
+)
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -28,10 +28,8 @@ github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
@@ -68,8 +66,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050 h1:S58HbKOjErL6jOafBHi/5TvtAsCozwGQ1mBGonIQpHA=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050/go.mod h1:FVYKQc+//BZ6eaAAXPsHzVaAKD4fkGytqcoRSvgpK9I=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20211120.1848 h1:VVP11xmoBcHbACKEXGDGt5DTjTWec7H1Dlpbxqr+7pQ=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20211120.1848/go.mod h1:xkWdP+DlOEQgm6PJdR7a53lvszgTWOnG7t0ZNzXnmns=
 github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
 github.com/crawlab-team/crawlab-db v0.1.1 h1:156h2fbbFKXAHs1mxprqRFC8zs2nrdyaG9JKG7patVw=
 github.com/crawlab-team/crawlab-db v0.1.1/go.mod h1:t0VidSjXKzQgACqNSQV5wusXncFtL6lGEiQTbLfNR04=
@@ -80,8 +78,8 @@ github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211009.1455 h1:jykwiu71Vy+bD4
 github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211009.1455/go.mod h1:W9Yee6xfesxoaqS5K1sF1I1zlH+i6xqwy4lyoBTOdkc=
 github.com/crawlab-team/crawlab-log v0.1.0 h1:0t+lZEojs3Vqb/bMkk2qs3I+1+XdwKG3pMTfeK5PZWM=
 github.com/crawlab-team/crawlab-log v0.1.0/go.mod h1:N8nTTKEbr9ZQSlmw0+HNB4ZAMQF4yVMaJLx8YhXvhNo=
-github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211103.2013 h1:kdlyHC4LFz8ANSk7W9HuzJn2HdoVnSR7ddsiYFsNel4=
-github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211103.2013/go.mod h1:G6Hnt/3255QCGHO5Q0xJe1AbJE7m5t65E0v7flRJBJM=
+github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211113.2048 h1:idtg0LELTXVBNVCP8ZAAm76cHtObTC6fyojH3Y+MTSo=
+github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211113.2048/go.mod h1:G6Hnt/3255QCGHO5Q0xJe1AbJE7m5t65E0v7flRJBJM=
 github.com/crawlab-team/go-trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg=
 github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8=
 github.com/crawlab-team/goseaweedfs v0.1.6/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA=
@@ -104,7 +102,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
@@ -157,7 +154,6 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.3.0 h1:nZU+7q+yJoFmwvNgv/LnPUkwPal62+b2xXj0AU1Es7o=
 github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
@@ -190,7 +186,6 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -252,7 +247,6 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -310,7 +304,6 @@ github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzV
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -339,7 +332,6 @@ github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8
 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFDU=
 github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
@@ -403,7 +395,6 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -427,12 +418,10 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty4DZO54I4FGqIpto=
 github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
@@ -531,7 +520,6 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY=
 go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
@@ -592,7 +580,6 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b h1:eB48h3HiRycXNy8E0Gf5e0hv7YT6Kt14L/D73G1fuwo=
 golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -648,7 +635,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
@@ -726,7 +712,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -738,7 +723,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
-gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
--- a/bin/docker-init.sh
+++ b/bin/docker-init.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then
+	# start master
+	/bin/bash /app/bin/docker-start-master.sh
+
+	# start crawlab
+	crawlab-server master
+else
+	# start crawlab
+	crawlab-server worker
+fi
--- a/bin/docker-start-master.sh
+++ b/bin/docker-start-master.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# start nginx
+service nginx start
+
+# start seaweedfs server
+seaweedfsDataPath=/data/seaweedfs
+if [ -e ${seaweedfsDataPath} ]; then
+	:
+else
+	mkdir -p ${seaweedfsDataPath}
+fi
+weed server \
+	-dir /data \
+	-master.dir ${seaweedfsDataPath} \
+	-volume.dir.idx ${seaweedfsDataPath} \
+	-ip localhost \
+	-volume.port 9999 \
+	-filer \
+	>> /var/log/weed.log 2>&1 &
--- a/changelog/v0.6.0-beta.20211120-zh.md
+++ b/changelog/v0.6.0-beta.20211120-zh.md
@@ -1,4 +1,4 @@
-# 更新日志 (v0.6.0-beta.20211116)
+# 更新日志 (v0.6.0-beta.20211120)

 ## 概览

--- a/changelog/v0.6.0-beta.20211120.md
+++ b/changelog/v0.6.0-beta.20211120.md
@@ -1,4 +1,4 @@
-# Change Log (v0.6.0-beta.20211116)
+# Change Log (v0.6.0-beta.20211120)

 ## Overview

@@ -30,4 +30,4 @@ This is the second beta release for the next major version v0.6.0 after the [fir

 ## What Next

-The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robostness and production-ready deployment.
+The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robustness and production-ready deployment.
--- a/docker_init.sh
+++ b/docker_init.sh
@@ -1,73 +0,0 @@
-#!/bin/bash
-
-# replace absolute api url to relative
-jspath=`ls /app/dist/js/index.*.js`
-sed -i "s?VUE_APP_API_BASE_URL?\/api?g" ${jspath}
-
-# replace default api path to new one
-# if [ "${CRAWLAB_API_ADDRESS}" = "" ]; 
-# then
-# 	:
-# else
-# 	jspath=`ls /app/dist/js/app.*.js`
-# 	sed -i "s?###CRAWLAB_API_ADDRESS###?${CRAWLAB_API_ADDRESS}?g" ${jspath}
-# fi
-
-# replace base url
-# if [ "${CRAWLAB_BASE_URL}" = "" ];
-# then
-# 	:
-# else
-# 	indexpath=/app/dist/index.html
-# 	sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath}
-# 	sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath}
-# fi
-
-#grant script 
-# chmod +x /app/backend/scripts/*.sh
-
-# install languages
-# if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_DOTNET}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ];
-# then
-# 	echo "installing languages"
-# 	echo "you can view log at /var/log/install.sh.log"
-# 	/bin/sh /app/backend/scripts/install.sh >> /var/log/install.sh.log 2>&1 &
-# fi
-
-# generate ssh
-ssh-keygen -q -t rsa -N "" -f ${HOME}/.ssh/id_rsa
-
-# ssh config
-touch ${HOME}/.ssh/config && chmod 600 ${HOME}/.ssh/config
-cat > ${HOME}/.ssh/config <<EOF
-Host *
-  StrictHostKeyChecking no
-EOF
-
-# start backend
-if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then
-	# start nginx
-	service nginx start
-
-	# start seaweedfs server
-	seaweedfsDataPath=/data/seaweedfs
-	if [ -e ${seaweedfsDataPath} ]; then
-		:
-	else
-		mkdir -p ${seaweedfsDataPath}
-	fi
-	weed server \
-		-dir /data \
-		-master.dir ${seaweedfsDataPath} \
-		-volume.dir.idx ${seaweedfsDataPath} \
-		-ip localhost \
-		-volume.port 9999 \
-		-filer \
-		>> /var/log/weed.log 2>&1 &
-
-	# start crawlab
-	crawlab-server master
-else
-	# start crawlab
-	crawlab-server worker
-fi
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -15,7 +15,7 @@
  },
  "license": "BSD-3-Clause",
  "dependencies": {
-    "crawlab-ui": "0.6.0-beta.20211113.2052",
+    "crawlab-ui": "0.6.0-beta.20211120.1838",
    "vue": "3.0.11",
    "vue-router": "^4.0.11"
  },
--- a/frontend/public/index.html
+++ b/frontend/public/index.html
@@ -7,6 +7,9 @@
  <link href="<%= BASE_URL %>favicon.ico" rel="icon">
  <link href="font-awesome.min.css" rel="stylesheet">
  <title><%= htmlWebpackPlugin.options.title %></title>
+  <script>
+  window.VUE_APP_API_BASE_URL = '<%= VUE_APP_API_BASE_URL %>'
+  </script>
  <script src="/js/vue3-sfc-loader.js"></script>
  <style>
      #loading-placeholder {
--- a/frontend/src/main.ts
+++ b/frontend/src/main.ts
@@ -2,7 +2,5 @@ import 'crawlab-ui/dist/crawlab-ui.css';
 import {createApp} from 'crawlab-ui';

 (async function () {
-  // @ts-ignore
-  window.VUE_APP_API_BASE_URL = process.env.VUE_APP_API_BASE_URL;
  await createApp();
 })();
--- a/workspace/docker-compose.yml
+++ b/workspace/docker-compose.yml
@@ -1,95 +1,66 @@
-version: "3.3"
+version: '3.3'
 services:
  master:
    build:
-      context: dockerfiles/golang
-    command: "air -c .air.conf"
+      context: ./dockerfiles/golang
+    command: /bin/bash -c "/app/bin/docker-start-master.sh && air -c .air.master.conf"
    volumes:
+      - ./.crawlab/master:/root/.crawlab
+      - ./.crawlab/go/pkg/mod:/go/pkg/mod
      - ../backend:/backend
-      - /backend/tmp
-    depends_on:
-      - mongo
-      - redis
-    ports:
-      - 8000:8000
+      - ../backend/go.mod.local:/backend/go.mod
+      - ../..:/libs/crawlab-team
+      - ../bin:/app/bin
+      - ../nginx:/etc/nginx/conf.d
+      - ../frontend/dist:/app/dist
    environment:
-      CRAWLAB_SPIDER_PATH: "/spiders"
-      CRAWLAB_SETTING_ENABLEDEMOSPIDERS: "Y"
+      CRAWLAB_NODE_MASTER: "Y"
+      CRAWLAB_NODE_NAME: "Master Node"
      CRAWLAB_MONGO_HOST: "mongo"
-      CRAWLAB_REDIS_ADDRESS: "redis"
-      CRAWLAB_SERVER_MASTER: "Y"
-      CRAWLAB_SERVER_REGISTER_TYPE: "customName"
-      CRAWLAB_SERVER_REGISTER_CUSTOMNODENAME: "master_1"
-      CRAWLAB_SERVER_PORT: 8000
-  worker-1:
-    build:
-      context: dockerfiles/golang
-    command: "air -c .air.conf"
+    ports:
+      - "9080:8080"
+      - "9000:8000"
    depends_on:
      - mongo
-      - redis
-    ports:
-      - 8001:8000
-    volumes:
-      - ../backend:/backend
-      - /backend/tmp

-    environment:
-      CRAWLAB_SPIDER_PATH: "/spiders"
-      CRAWLAB_MONGO_HOST: "mongo"
-      CRAWLAB_REDIS_ADDRESS: "redis"
-      CRAWLAB_SERVER_MASTER: "N"
-      CRAWLAB_SERVER_PORT: 8001
-      CRAWLAB_SERVER_REGISTER_TYPE: "customName"
-      CRAWLAB_SERVER_REGISTER_CUSTOMNODENAME: "worker_1"
-  worker-2:
+  worker01:
    build:
-      context: dockerfiles/golang
-    command: "air -c .air.conf"
-    depends_on:
-      - mongo
-      - redis
-    ports:
-      - 8002:8000
+      context: ./dockerfiles/golang
+    command: "air -c .air.worker.conf"
+    environment:
+      CRAWLAB_NODE_MASTER: "N"
+      CRAWLAB_NODE_NAME: "Worker Node 01"
+      CRAWLAB_GRPC_ADDRESS: "master"
+      CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
    volumes:
+      - ./.crawlab/worker01:/root/.crawlab
+      - ./.crawlab/go/pkg/mod:/go/pkg/mod
      - ../backend:/backend
-    environment:
-      CRAWLAB_SPIDER_PATH: "/spiders"
-      CRAWLAB_MONGO_HOST: "mongo"
-      CRAWLAB_REDIS_ADDRESS: "redis"
-      CRAWLAB_SERVER_MASTER: "N"
-      CRAWLAB_SERVER_PORT: 8002
-      CRAWLAB_SERVER_REGISTER_TYPE: "customName"
-      CRAWLAB_SERVER_REGISTER_CUSTOMNODENAME: "worker_2"
-  ui:
-    build:
-      context: ./dockerfiles/node
-    container_name: crawlab_frontend
-    ports:
-      - 8080:8080
-    volumes:
-      - ../frontend:/frontend
-      - /frontend/node_modules
-    command:
-      - /bin/sh
-      - -c
-      - |
-        yarn install
-        yarn run serve
+      - ../backend/go.mod.local:/backend/go.mod
+      - ../..:/libs/crawlab-team
    depends_on:
      - master
+
+  worker02:
+    build:
+      context: ./dockerfiles/golang
+    command: "air -c .air.worker.conf"
+    environment:
+      CRAWLAB_NODE_MASTER: "N"
+      CRAWLAB_NODE_NAME: "Worker Node 01"
+      CRAWLAB_GRPC_ADDRESS: "master"
+      CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
+    volumes:
+      - ./.crawlab/worker02:/root/.crawlab
+      - ./.crawlab/go/pkg/mod:/go/pkg/mod
+      - ../backend:/backend
+      - ../backend/go.mod.local:/backend/go.mod
+      - ../..:/libs/crawlab-team
+    depends_on:
+      - master
+
  mongo:
-    image: mongo:latest
+    image: mongo:4
    restart: always
-    # volumes:
-    #   - "/opt/crawlab/mongo/data/db:/data/db"  # make data persistent 持久化
    ports:
-      - "27017:27017"  # expose port to host machine 暴露接口到宿主机
-  redis:
-    image: redis:latest
-    restart: always
-    # command: redis-server --requirepass "password" # set redis password 设置 Redis 密码
-    # volumes:
-    #   - "/opt/crawlab/redis/data:/data"  # make data persistent 持久化
-    ports:
-      - "6379:6379"  # expose port to host machine 暴露接口到宿主机
+      - "28017:27017"
--- a/workspace/dockerfiles/golang/Dockerfile
+++ b/workspace/dockerfiles/golang/Dockerfile
@@ -1,26 +1,40 @@
-FROM golang:buster
+FROM golang:1.16
+
 RUN go env -w GOPROXY=https://goproxy.io,https://goproxy.cn && \
    go env -w GO111MODULE="on"
+
 WORKDIR /tools
 RUN go get github.com/cosmtrek/air
+
 WORKDIR /backend
 RUN rm -rf /tools
+
 # set as non-interactive
 ENV DEBIAN_FRONTEND noninteractive

-# set CRAWLAB_IS_DOCKER
-ENV CRAWLAB_IS_DOCKER Y
 # install packages
 RUN chmod 777 /tmp \
-    && sed -i 's#http://deb.debian.org#https://mirrors.tuna.tsinghua.edu.cn#g' /etc/apt/sources.list \
 	&& apt-get update \
-	&& apt-get install -y curl net-tools iputils-ping ntp ntpdate python3 python3-pip dumb-init  \
+	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc
+
+# install python
+RUN apt-get install -y python3 python3-pip \
 	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
 	&& ln -s /usr/bin/python3 /usr/local/bin/python

-RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
+# install golang
+RUN curl -OL https://storage.googleapis.com/golang/go1.16.7.linux-amd64.tar.gz \
+	&& tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \
+	&& ln -s /usr/local/go/bin/go /usr/local/bin/go
+
+# install seaweedfs
+RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.76/linux_amd64.tar.gz \
+  && tar -zxf linux_amd64.tar.gz \
+  && cp weed /usr/local/bin
+
 # install backend
-RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash
-RUN mkdir /spiders && chmod -R 0755 /spiders
+RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
+RUN pip install crawlab-sdk==0.6.b20211024-1207
+
 VOLUME /backend
 EXPOSE 8080
--- a/workspace/dockerfiles/node/Dockerfile
+++ b/workspace/dockerfiles/node/Dockerfile
@@ -1,5 +1,4 @@
-FROM node:latest
+FROM node:12
 WORKDIR frontend
 ENV SASS_BINARY_SITE=https://npm.taobao.org/mirrors/node-sass/
 RUN npm config set registry "http://registry.npm.taobao.org"
-