diff --git a/.gitignore b/.gitignore index 47cefbd8..5e091d2d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ tmp/ _book/ *.lock vendor/ +.crawlab diff --git a/Dockerfile b/Dockerfile index aac69e14..b64ceebf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.15 AS backend-build +FROM golang:1.16 AS backend-build WORKDIR /go/src/app COPY ./backend . @@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207 # add files COPY ./backend/conf /app/backend/conf COPY ./nginx /app/nginx -COPY ./docker_init.sh /app/docker_init.sh +COPY ./bin /app/bin # copy backend files RUN mkdir -p /opt/bin @@ -84,4 +84,4 @@ EXPOSE 8080 EXPOSE 8000 # start backend -CMD ["/bin/bash", "/app/docker_init.sh"] +CMD ["/bin/bash", "/app/bin/docker-init.sh"] diff --git a/Dockerfile.cn b/Dockerfile.cn index 0449e527..edf54b70 100644 --- a/Dockerfile.cn +++ b/Dockerfile.cn @@ -1,4 +1,4 @@ -FROM golang:1.15 AS backend-build +FROM golang:1.16 AS backend-build WORKDIR /go/src/app COPY ./backend . @@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207 # add files COPY ./backend/conf /app/backend/conf COPY ./nginx /app/nginx -COPY ./docker_init.sh /app/docker_init.sh +COPY ./bin /app/bin # copy backend files RUN mkdir -p /opt/bin @@ -84,4 +84,4 @@ EXPOSE 8080 EXPOSE 8000 # start backend -CMD ["/bin/bash", "/app/docker_init.sh"] +CMD ["/bin/bash", "/app/bin/docker-init.sh"] diff --git a/Dockerfile.local b/Dockerfile.local index 11d60e38..d04207fa 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,4 +1,4 @@ -FROM golang:1.15 AS backend-build +FROM golang:1.16 AS backend-build WORKDIR /go/src/app COPY ./backend . @@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207 # add files COPY ./backend/conf /app/backend/conf COPY ./nginx /app/nginx -COPY ./docker_init.sh /app/docker_init.sh +COPY ./bin /app/bin # copy backend files RUN mkdir -p /opt/bin @@ -84,4 +84,4 @@ EXPOSE 8080 EXPOSE 8000 # start backend -CMD ["/bin/bash", "/app/docker_init.sh"] +CMD ["/bin/bash", "/app/bin/docker-init.sh"] diff --git a/backend/.air.conf b/backend/.air.master.conf similarity index 91% rename from backend/.air.conf rename to backend/.air.master.conf index a6d35204..47c2ee4d 100644 --- a/backend/.air.conf +++ b/backend/.air.master.conf @@ -7,17 +7,17 @@ tmp_dir = "/tmp" [build] # Just plain old shell command. You could use `make` as well. -cmd = "go build -o ../tmp/main ./ " +cmd = "go build -o ../tmp/main ./ " # Binary file yields from `cmd`. bin = "../tmp/main" # Customize binary. -full_bin = "../tmp/main start" +full_bin = "../tmp/main master" # Watch these filename extensions. include_ext = ["go", "tpl", "tmpl", "html"] # Ignore these filename extensions or directories. exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"] # Watch these directories if you specified. -include_dir = [] +include_dir = ["../libs"] # Exclude files. exclude_file = [] # This log file places in your tmp_dir. diff --git a/backend/.air.worker.conf b/backend/.air.worker.conf new file mode 100644 index 00000000..6198cadc --- /dev/null +++ b/backend/.air.worker.conf @@ -0,0 +1,47 @@ +# Config file for [Air](https://github.com/cosmtrek/air) in TOML format + +# Working directory +# . or absolute path, please note that the directories following must be under root. +root = "." +tmp_dir = "/tmp" + +[build] +# Just plain old shell command. You could use `make` as well. +cmd = "go build -o ../tmp/main ./ " +# Binary file yields from `cmd`. +bin = "../tmp/main" +# Customize binary. +full_bin = "../tmp/main worker" +# Watch these filename extensions. +include_ext = ["go", "tpl", "tmpl", "html"] +# Ignore these filename extensions or directories. +exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"] +# Watch these directories if you specified. +include_dir = ["../libs"] +# Exclude files. +exclude_file = [] +# This log file places in your tmp_dir. +log = "air.log" +# It's not necessary to trigger build each time file changes if it's too frequent. +delay = 1000 # ms +# Stop running old binary when build errors occur. +stop_on_error = true +# Send Interrupt signal before killing process (windows does not support this feature) +send_interrupt = false +# Delay after sending Interrupt signal +kill_delay = 500 # ms + +[log] +# Show log time +time = false + +[color] +# Customize each part's color. If no color found, use the raw app log. +main = "magenta" +watcher = "cyan" +build = "yellow" +runner = "green" + +[misc] +# Delete tmp directory on exit +clean_on_exit = true \ No newline at end of file diff --git a/backend/go.mod b/backend/go.mod index 61a6072c..dded6b1f 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,10 +1,10 @@ module crawlab -go 1.15 +go 1.16 require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050 + github.com/crawlab-team/crawlab-core v0.6.0-beta.20211120.1848 github.com/crawlab-team/go-trace v0.1.0 github.com/gin-gonic/gin v1.6.3 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.mod.local b/backend/go.mod.local new file mode 100644 index 00000000..1bfca0b3 --- /dev/null +++ b/backend/go.mod.local @@ -0,0 +1,21 @@ +module crawlab + +go 1.15 + +replace ( + github.com/crawlab-team/crawlab-core => /libs/crawlab-team/crawlab-core + github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs + github.com/crawlab-team/crawlab-fs => /libs/crawlab-team/crawlab-fs + github.com/crawlab-team/crawlab-db => /libs/crawlab-team/crawlab-db +) + +require ( + github.com/apex/log v1.9.0 + github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050 + github.com/crawlab-team/go-trace v0.1.0 + github.com/gin-gonic/gin v1.6.3 + github.com/spf13/cobra v1.1.3 + github.com/spf13/viper v1.7.1 + go.mongodb.org/mongo-driver v1.6.0 // indirect + go.uber.org/dig v1.10.0 +) diff --git a/backend/go.sum b/backend/go.sum index e47485a8..5397410f 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -28,10 +28,8 @@ github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= @@ -68,8 +66,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050 h1:S58HbKOjErL6jOafBHi/5TvtAsCozwGQ1mBGonIQpHA= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050/go.mod h1:FVYKQc+//BZ6eaAAXPsHzVaAKD4fkGytqcoRSvgpK9I= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20211120.1848 h1:VVP11xmoBcHbACKEXGDGt5DTjTWec7H1Dlpbxqr+7pQ= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20211120.1848/go.mod h1:xkWdP+DlOEQgm6PJdR7a53lvszgTWOnG7t0ZNzXnmns= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.1.1 h1:156h2fbbFKXAHs1mxprqRFC8zs2nrdyaG9JKG7patVw= github.com/crawlab-team/crawlab-db v0.1.1/go.mod h1:t0VidSjXKzQgACqNSQV5wusXncFtL6lGEiQTbLfNR04= @@ -80,8 +78,8 @@ github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211009.1455 h1:jykwiu71Vy+bD4 github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211009.1455/go.mod h1:W9Yee6xfesxoaqS5K1sF1I1zlH+i6xqwy4lyoBTOdkc= github.com/crawlab-team/crawlab-log v0.1.0 h1:0t+lZEojs3Vqb/bMkk2qs3I+1+XdwKG3pMTfeK5PZWM= github.com/crawlab-team/crawlab-log v0.1.0/go.mod h1:N8nTTKEbr9ZQSlmw0+HNB4ZAMQF4yVMaJLx8YhXvhNo= -github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211103.2013 h1:kdlyHC4LFz8ANSk7W9HuzJn2HdoVnSR7ddsiYFsNel4= -github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211103.2013/go.mod h1:G6Hnt/3255QCGHO5Q0xJe1AbJE7m5t65E0v7flRJBJM= +github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211113.2048 h1:idtg0LELTXVBNVCP8ZAAm76cHtObTC6fyojH3Y+MTSo= +github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211113.2048/go.mod h1:G6Hnt/3255QCGHO5Q0xJe1AbJE7m5t65E0v7flRJBJM= github.com/crawlab-team/go-trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg= github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= github.com/crawlab-team/goseaweedfs v0.1.6/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= @@ -104,7 +102,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -157,7 +154,6 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.3.0 h1:nZU+7q+yJoFmwvNgv/LnPUkwPal62+b2xXj0AU1Es7o= github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= @@ -190,7 +186,6 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -252,7 +247,6 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -310,7 +304,6 @@ github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzV github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -339,7 +332,6 @@ github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFDU= github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -403,7 +395,6 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -427,12 +418,10 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty4DZO54I4FGqIpto= github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= @@ -531,7 +520,6 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= @@ -592,7 +580,6 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b h1:eB48h3HiRycXNy8E0Gf5e0hv7YT6Kt14L/D73G1fuwo= golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -648,7 +635,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= @@ -726,7 +712,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -738,7 +723,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/bin/docker-init.sh b/bin/docker-init.sh new file mode 100755 index 00000000..f15755db --- /dev/null +++ b/bin/docker-init.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then + # start master + /bin/bash /app/bin/docker-start-master.sh + + # start crawlab + crawlab-server master +else + # start crawlab + crawlab-server worker +fi diff --git a/bin/docker-start-master.sh b/bin/docker-start-master.sh new file mode 100755 index 00000000..339640fd --- /dev/null +++ b/bin/docker-start-master.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# start nginx +service nginx start + +# start seaweedfs server +seaweedfsDataPath=/data/seaweedfs +if [ -e ${seaweedfsDataPath} ]; then + : +else + mkdir -p ${seaweedfsDataPath} +fi +weed server \ + -dir /data \ + -master.dir ${seaweedfsDataPath} \ + -volume.dir.idx ${seaweedfsDataPath} \ + -ip localhost \ + -volume.port 9999 \ + -filer \ + >> /var/log/weed.log 2>&1 & diff --git a/changelog/v0.6.0-beta.20211116-zh.md b/changelog/v0.6.0-beta.20211120-zh.md similarity index 97% rename from changelog/v0.6.0-beta.20211116-zh.md rename to changelog/v0.6.0-beta.20211120-zh.md index 1479932e..50632cd8 100644 --- a/changelog/v0.6.0-beta.20211116-zh.md +++ b/changelog/v0.6.0-beta.20211120-zh.md @@ -1,4 +1,4 @@ -# 更新日志 (v0.6.0-beta.20211116) +# 更新日志 (v0.6.0-beta.20211120) ## 概览 diff --git a/changelog/v0.6.0-beta.20211116.md b/changelog/v0.6.0-beta.20211120.md similarity index 94% rename from changelog/v0.6.0-beta.20211116.md rename to changelog/v0.6.0-beta.20211120.md index f55e18f5..d3c134e0 100644 --- a/changelog/v0.6.0-beta.20211116.md +++ b/changelog/v0.6.0-beta.20211120.md @@ -1,4 +1,4 @@ -# Change Log (v0.6.0-beta.20211116) +# Change Log (v0.6.0-beta.20211120) ## Overview @@ -30,4 +30,4 @@ This is the second beta release for the next major version v0.6.0 after the [fir ## What Next -The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robostness and production-ready deployment. +The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robustness and production-ready deployment. diff --git a/docker_init.sh b/docker_init.sh deleted file mode 100755 index ad3d9815..00000000 --- a/docker_init.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -# replace absolute api url to relative -jspath=`ls /app/dist/js/index.*.js` -sed -i "s?VUE_APP_API_BASE_URL?\/api?g" ${jspath} - -# replace default api path to new one -# if [ "${CRAWLAB_API_ADDRESS}" = "" ]; -# then -# : -# else -# jspath=`ls /app/dist/js/app.*.js` -# sed -i "s?###CRAWLAB_API_ADDRESS###?${CRAWLAB_API_ADDRESS}?g" ${jspath} -# fi - -# replace base url -# if [ "${CRAWLAB_BASE_URL}" = "" ]; -# then -# : -# else -# indexpath=/app/dist/index.html -# sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath} -# sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath} -# fi - -#grant script -# chmod +x /app/backend/scripts/*.sh - -# install languages -# if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_DOTNET}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ]; -# then -# echo "installing languages" -# echo "you can view log at /var/log/install.sh.log" -# /bin/sh /app/backend/scripts/install.sh >> /var/log/install.sh.log 2>&1 & -# fi - -# generate ssh -ssh-keygen -q -t rsa -N "" -f ${HOME}/.ssh/id_rsa - -# ssh config -touch ${HOME}/.ssh/config && chmod 600 ${HOME}/.ssh/config -cat > ${HOME}/.ssh/config <> /var/log/weed.log 2>&1 & - - # start crawlab - crawlab-server master -else - # start crawlab - crawlab-server worker -fi diff --git a/frontend/package.json b/frontend/package.json index c4165cd8..91e0aa03 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -15,7 +15,7 @@ }, "license": "BSD-3-Clause", "dependencies": { - "crawlab-ui": "0.6.0-beta.20211113.2052", + "crawlab-ui": "0.6.0-beta.20211120.1838", "vue": "3.0.11", "vue-router": "^4.0.11" }, diff --git a/frontend/public/index.html b/frontend/public/index.html index e71e5256..1196d73a 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -7,6 +7,9 @@ <%= htmlWebpackPlugin.options.title %> +