diff --git a/.gitignore b/.gitignore index 1755f6f1..47cefbd8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,128 +1,9 @@ .idea/ .vscode/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ - -# node_modules -node_modules/ - -# egg-info -*.egg-info - -# .DS_Store .DS_Store - -.docks -.docs - +node_modules/ logs/ tmp/ _book/ -.idea *.lock - -backend/spiders -spiders/*.zip - vendor/ diff --git a/Dockerfile b/Dockerfile index d0668687..aac69e14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,9 +16,6 @@ WORKDIR /app RUN rm /app/.npmrc # install frontend -#RUN npm config set unsafe-perm true -#RUN npm install -g yarn && yarn install - RUN yarn install && yarn run build:docker # images @@ -33,18 +30,26 @@ ENV CRAWLAB_IS_DOCKER Y # install packages RUN chmod 777 /tmp \ && apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget dumb-init cloc \ + && apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc + +# install python +RUN apt-get install -y python3 python3-pip \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python +# install golang +RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \ + && tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \ + && ln -s /usr/local/go/bin/go /usr/local/bin/go + # install seaweedfs -RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.59/linux_amd64.tar.gz \ +RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.76/linux_amd64.tar.gz \ && tar -zxf linux_amd64.tar.gz \ && cp weed /usr/local/bin # install backend RUN pip install scrapy pymongo bs4 requests -RUN pip install crawlab-sdk==0.6.b20210729-1634 +RUN pip install crawlab-sdk==0.6.b20211024-1207 # add files COPY ./backend/conf /app/backend/conf diff --git a/Dockerfile.cn b/Dockerfile.cn index b7a7bf4b..0449e527 100644 --- a/Dockerfile.cn +++ b/Dockerfile.cn @@ -16,9 +16,6 @@ WORKDIR /app #RUN rm /app/.npmrc # install frontend -#RUN npm config set unsafe-perm true -#RUN npm install -g yarn && yarn install - RUN yarn install && yarn run build:docker # images @@ -33,18 +30,26 @@ ENV CRAWLAB_IS_DOCKER Y # install packages RUN chmod 777 /tmp \ && apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget dumb-init cloc \ + && apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc + +# install python +RUN apt-get install -y python3 python3-pip \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python +# install golang +RUN curl -OL https://storage.googleapis.com/golang/go1.16.7.linux-amd64.tar.gz \ + && tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \ + && ln -s /usr/local/go/bin/go /usr/local/bin/go + # install seaweedfs -RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.59/linux_amd64.tar.gz \ +RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.76/linux_amd64.tar.gz \ && tar -zxf linux_amd64.tar.gz \ && cp weed /usr/local/bin # install backend RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple -RUN pip install crawlab-sdk==0.6.b20210729-1634 +RUN pip install crawlab-sdk==0.6.b20211024-1207 # add files COPY ./backend/conf /app/backend/conf diff --git a/Dockerfile.local b/Dockerfile.local index b7a7bf4b..11d60e38 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -16,9 +16,6 @@ WORKDIR /app #RUN rm /app/.npmrc # install frontend -#RUN npm config set unsafe-perm true -#RUN npm install -g yarn && yarn install - RUN yarn install && yarn run build:docker # images @@ -33,18 +30,26 @@ ENV CRAWLAB_IS_DOCKER Y # install packages RUN chmod 777 /tmp \ && apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget dumb-init cloc \ + && apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc + +# install python +RUN apt-get install -y python3 python3-pip \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python +# install golang +RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \ + && tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \ + && ln -s /usr/local/go/bin/go /usr/local/bin/go + # install seaweedfs -RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.59/linux_amd64.tar.gz \ +RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.76/linux_amd64.tar.gz \ && tar -zxf linux_amd64.tar.gz \ && cp weed /usr/local/bin # install backend RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple -RUN pip install crawlab-sdk==0.6.b20210729-1634 +RUN pip install crawlab-sdk==0.6.b20211024-1207 # add files COPY ./backend/conf /app/backend/conf diff --git a/backend/Dockerfile b/backend/Dockerfile deleted file mode 100644 index 4cbfda47..00000000 --- a/backend/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM golang:1.12-alpine AS build - -WORKDIR /go/src/app -COPY . . - -ENV GO111MODULE on -ENV GOPROXY https://mirrors.aliyun.com/goproxy/ - -RUN go mod vendor -RUN go install -v ./... - -FROM alpine:latest -WORKDIR /root -COPY --from=build /go/src/app . -COPY --from=build /go/bin/crawlab /usr/local/bin - -EXPOSE 8000 -CMD ["crawlab"] diff --git a/backend/cmd/master.go b/backend/cmd/master.go index d85e3e51..478c1450 100644 --- a/backend/cmd/master.go +++ b/backend/cmd/master.go @@ -2,30 +2,11 @@ package cmd import ( "crawlab/apps" - "fmt" - "github.com/crawlab-team/crawlab-core/entity" "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var ( - runOnMaster bool - masterConfigPath string - masterGrpcAddress string - masterGrpcAuthKey string ) func init() { rootCmd.AddCommand(masterCmd) - - masterCmd.PersistentFlags().StringVarP(&masterConfigPath, "config-path", "c", "", "Config path of master node") - _ = viper.BindPFlag("configPath", masterCmd.PersistentFlags().Lookup("configPath")) - - masterCmd.PersistentFlags().StringVarP(&masterGrpcAddress, "grpc-address", "g", "", "gRPC address of master node") - _ = viper.BindPFlag("grpcAddress", masterCmd.PersistentFlags().Lookup("grpcAddress")) - - masterCmd.PersistentFlags().StringVarP(&masterGrpcAuthKey, "grpc-auth-key", "a", "", "gRPC auth key of master node") - _ = viper.BindPFlag("grpcAuthKey", masterCmd.PersistentFlags().Lookup("grpcAuthKey")) } var masterCmd = &cobra.Command{ @@ -37,23 +18,6 @@ which runs api and assign tasks to worker nodes`, Run: func(cmd *cobra.Command, args []string) { // options var opts []apps.MasterOption - if masterConfigPath != "" { - opts = append(opts, apps.WithMasterConfigPath(masterConfigPath)) - viper.Set("config.path", masterConfigPath) - } - opts = append(opts, apps.WithRunOnMaster(runOnMaster)) - if masterGrpcAddress != "" { - address, err := entity.NewAddressFromString(masterGrpcAddress) - if err != nil { - fmt.Println(fmt.Sprintf("invalid grpc-address: %s", masterGrpcAddress)) - } - opts = append(opts, apps.WithMasterGrpcAddress(address)) - viper.Set("grpc.address", masterGrpcAddress) - viper.Set("grpc.server.address", masterGrpcAddress) - } - if masterGrpcAuthKey != "" { - viper.Set("grpc.authKey", masterGrpcAuthKey) - } // app master := apps.NewMaster(opts...) diff --git a/backend/cmd/root.go b/backend/cmd/root.go index 11285130..8c6c322f 100644 --- a/backend/cmd/root.go +++ b/backend/cmd/root.go @@ -50,6 +50,7 @@ func initConfig() { replacer := strings.NewReplacer(".", "_") viper.SetEnvKeyReplacer(replacer) + // read config file if err := viper.ReadInConfig(); err == nil { fmt.Println("Using config file:", viper.ConfigFileUsed()) } diff --git a/backend/cmd/worker.go b/backend/cmd/worker.go index 490cdd47..d32b4eba 100644 --- a/backend/cmd/worker.go +++ b/backend/cmd/worker.go @@ -2,29 +2,11 @@ package cmd import ( "crawlab/apps" - "fmt" - "github.com/crawlab-team/crawlab-core/entity" "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -var ( - workerConfigPath string - workerGrpcAddress string - workerGrpcAuthKey string ) func init() { rootCmd.AddCommand(workerCmd) - - workerCmd.PersistentFlags().StringVarP(&workerConfigPath, "config-path", "c", "", "Config path of worker node") - _ = viper.BindPFlag("configPath", workerCmd.PersistentFlags().Lookup("configPath")) - - workerCmd.PersistentFlags().StringVarP(&workerGrpcAddress, "grpc-address", "g", "", "gRPC address of worker node") - _ = viper.BindPFlag("grpcAddress", workerCmd.PersistentFlags().Lookup("grpcAddress")) - - workerCmd.PersistentFlags().StringVarP(&workerGrpcAuthKey, "grpc-auth-key", "a", "", "gRPC auth key of worker node") - _ = viper.BindPFlag("grpcAuthKey", workerCmd.PersistentFlags().Lookup("grpcAuthKey")) } var workerCmd = &cobra.Command{ @@ -37,22 +19,6 @@ assigned by the master node`, Run: func(cmd *cobra.Command, args []string) { // options var opts []apps.WorkerOption - if workerConfigPath != "" { - opts = append(opts, apps.WithWorkerConfigPath(workerConfigPath)) - viper.Set("config.path", workerConfigPath) - } - if workerGrpcAddress != "" { - address, err := entity.NewAddressFromString(workerGrpcAddress) - if err != nil { - fmt.Println(fmt.Sprintf("invalid grpc-address: %s", workerGrpcAddress)) - return - } - opts = append(opts, apps.WithWorkerGrpcAddress(address)) - viper.Set("grpc.address", workerGrpcAddress) - } - if workerGrpcAuthKey != "" { - viper.Set("grpc.authKey", workerGrpcAuthKey) - } // app master := apps.NewWorker(opts...) diff --git a/backend/go.mod b/backend/go.mod index acca7cfe..61a6072c 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,7 +4,7 @@ go 1.15 require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20210811.1634 + github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050 github.com/crawlab-team/go-trace v0.1.0 github.com/gin-gonic/gin v1.6.3 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.sum b/backend/go.sum index 8b3b71be..e47485a8 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -16,9 +16,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -26,20 +24,17 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v1.2.0 h1:noJEYkMQVlFCEAc+2ma5YyRhlfjcWfZqk5sBRYozdyM= github.com/StackExchange/wmi v1.2.0/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw= github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= @@ -72,28 +67,26 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/crawlab-team/crawlab-core v0.0.1 h1:tZ9Rlji5L5uoZ6nRZFqnSGmEl5TdmdxAyxueJDKfUl8= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20210811.1634 h1:rUCWl/AwGczVIx54StgAeu3pOVN9n4zUWcN9//7Juu8= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20210811.1634/go.mod h1:dvqfN3ZjLZQSxXYkHFFPEbgxZNHqad+yFhZv++Qg4tQ= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050 h1:S58HbKOjErL6jOafBHi/5TvtAsCozwGQ1mBGonIQpHA= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050/go.mod h1:FVYKQc+//BZ6eaAAXPsHzVaAKD4fkGytqcoRSvgpK9I= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.1.1 h1:156h2fbbFKXAHs1mxprqRFC8zs2nrdyaG9JKG7patVw= github.com/crawlab-team/crawlab-db v0.1.1/go.mod h1:t0VidSjXKzQgACqNSQV5wusXncFtL6lGEiQTbLfNR04= github.com/crawlab-team/crawlab-fs v0.0.0/go.mod h1:k2VXprQspLAmbgO5sSpqMjg/xP4iKDkW4RyTWY8eTZM= -github.com/crawlab-team/crawlab-fs v0.1.0 h1:iKSJJY4Wvea8Qss+zC/tLiZ371VeV75Z3cuqlsxydzY= -github.com/crawlab-team/crawlab-fs v0.1.0/go.mod h1:dOE0TeWPDz9krwzt1H72rjj0Fn/aHe53yn7GoOZHD0s= -github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20210811.1628 h1:VcW4n+EvmkbG1UMJ0xPDDyVS+lzKywIcrgFG+VJw1RA= -github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20210811.1628/go.mod h1:W9Yee6xfesxoaqS5K1sF1I1zlH+i6xqwy4lyoBTOdkc= +github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940 h1:KFZ39oe/QyhhRhXYZSmzDZl3L/JUEPkiiaf4+/iuboY= +github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940/go.mod h1:dA1G6xeiClbTMkjRuoagGrcKfQ97jJZRAhZUSwrKdoI= +github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211009.1455 h1:jykwiu71Vy+bD4taRQBkHhPureFTSAhpxuK5WdaWj/A= +github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211009.1455/go.mod h1:W9Yee6xfesxoaqS5K1sF1I1zlH+i6xqwy4lyoBTOdkc= github.com/crawlab-team/crawlab-log v0.1.0 h1:0t+lZEojs3Vqb/bMkk2qs3I+1+XdwKG3pMTfeK5PZWM= github.com/crawlab-team/crawlab-log v0.1.0/go.mod h1:N8nTTKEbr9ZQSlmw0+HNB4ZAMQF4yVMaJLx8YhXvhNo= -github.com/crawlab-team/crawlab-vcs v0.1.0 h1:LjtKOOFzx1o7vvgGppC7jt/8lznyvFwwXBYggbSW9+4= -github.com/crawlab-team/crawlab-vcs v0.1.0/go.mod h1:G6Hnt/3255QCGHO5Q0xJe1AbJE7m5t65E0v7flRJBJM= +github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211103.2013 h1:kdlyHC4LFz8ANSk7W9HuzJn2HdoVnSR7ddsiYFsNel4= +github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211103.2013/go.mod h1:G6Hnt/3255QCGHO5Q0xJe1AbJE7m5t65E0v7flRJBJM= github.com/crawlab-team/go-trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg= github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= github.com/crawlab-team/goseaweedfs v0.1.6/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= -github.com/crawlab-team/goseaweedfs v0.2.0/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20210725.1917 h1:Kb8AErE3357UO0jPf8Q2wqG/qcmL0hKDwPMaOZ/JjcY= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20210725.1917/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= +github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936 h1:c4SgTj2baDqD2UYa1eCpj3ukOF3mXOjvOCP4cWwgfyw= +github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -131,7 +124,6 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= @@ -165,6 +157,7 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.3.0 h1:nZU+7q+yJoFmwvNgv/LnPUkwPal62+b2xXj0AU1Es7o= github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= @@ -274,7 +267,6 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= @@ -284,7 +276,6 @@ github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= @@ -360,7 +351,6 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -387,7 +377,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE= github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE= @@ -469,9 +458,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= -github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -488,6 +475,8 @@ github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuI github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= github.com/swaggo/swag v1.6.6/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc= +github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= @@ -605,6 +594,8 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b h1:eB48h3HiRycXNy8E0Gf5e0hv7YT6Kt14L/D73G1fuwo= +golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -647,6 +638,7 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -658,6 +650,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -733,7 +728,6 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -749,13 +743,9 @@ gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkp gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA= gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= diff --git a/backend/test/config-master.json b/backend/test/config-master.json index 98ead88c..411e1a75 100644 --- a/backend/test/config-master.json +++ b/backend/test/config-master.json @@ -1,7 +1,7 @@ { "key": "master", "is_master": true, - "name": "master", + "name": "Master Node", "ip": "", "mac": "", "hostname": "", diff --git a/backend/test/config-worker.json b/backend/test/config-worker-01.json similarity index 69% rename from backend/test/config-worker.json rename to backend/test/config-worker-01.json index f88c1630..7b43497d 100644 --- a/backend/test/config-worker.json +++ b/backend/test/config-worker-01.json @@ -1,7 +1,7 @@ { - "key": "worker", + "key": "worker-01", "is_master": false, - "name": "worker", + "name": "Worker Node 01", "ip": "", "mac": "", "hostname": "", diff --git a/backend/test/config-worker-02.json b/backend/test/config-worker-02.json new file mode 100644 index 00000000..bdd2e527 --- /dev/null +++ b/backend/test/config-worker-02.json @@ -0,0 +1,10 @@ +{ + "key": "worker-02", + "is_master": false, + "name": "Worker Node 02", + "ip": "", + "mac": "", + "hostname": "", + "description": "", + "auth_key": "Crawlab2021!" +} \ No newline at end of file diff --git a/changelog/v0.6.0-beta.20210803-zh.md b/changelog/v0.6.0-beta.20210803-zh.md index e65b7fca..2aa632a5 100644 --- a/changelog/v0.6.0-beta.20210803-zh.md +++ b/changelog/v0.6.0-beta.20210803-zh.md @@ -37,7 +37,7 @@ - [ ] **Git 集成**. 将作为插件存在。 - [ ] **Scrapy 集成**. 将作为插件存在。 - [ ] **消息通知**. 将作为插件存在。 -- [ ] **关联人物**. 如果任务执行模式为 “所有节点” 或 “指定节点”,那么将会有主任务和子任务之分。 +- [ ] **关联任务**. 如果任务执行模式为 “所有节点” 或 “指定节点”,那么将会有主任务和子任务之分。 - [ ] **Crontab 编辑器**. 可视化 Crontab 编辑的前端组件。 - [ ] **结果去重**. - [ ] **环境变量**. diff --git a/changelog/v0.6.0-beta.20211116-zh.md b/changelog/v0.6.0-beta.20211116-zh.md new file mode 100644 index 00000000..1479932e --- /dev/null +++ b/changelog/v0.6.0-beta.20211116-zh.md @@ -0,0 +1,34 @@ +# 更新日志 (v0.6.0-beta.20211116) + +## 概览 + +这是下个重大版本 v0.6.0 的第二次 beta 发布(参考 [第一次 beta 版本](https://github.com/crawlab-team/crawlab/releases/tag/v0.6.0-beta.20210803))。伴随着更多功能和优化,正式版本 v0.6.0 将会很快发布了。 + +## 升级优化 + +#### 后端 + +- [x] **插件框架**. **Crawlab 插件框架 (CPF)** 已发布. 详情请参考 [这里](https://docs-next.crawlab.cn/zh/guide/use-crawlab/basic-concepts/plugin.html). +- [x] **Git 集成**. Git 集成被作为内置功能. +- [x] **Scrapy 集成**. Scrapy 集成以插件形式存在,插件为 [spider-assistant](https://github.com/crawlab-team/plugin-spider-assistant). +- [x] **依赖集成**. Dependency 集成以插件形式存在,插件为 [dependency](https://github.com/crawlab-team/plugin-dependency). +- [x] **消息通知**. 消息通知功能以插件形式存在,插件为 [notification](https://github.com/crawlab-team/plugin-notification). +- [x] **文档网站**. 搭建 [文档网站](https://docs-next.crawlab.cn). + +#### 前端 +- **Bug 修复**. + +#### 待完成 +- [ ] **关联任务**. 如果任务执行模式为 “所有节点” 或 “指定节点”,那么将会有主任务和子任务之分。 +- [ ] **Crontab 编辑器**. 可视化 Crontab 编辑的前端组件。 +- [ ] **结果去重**. +- [ ] **环境变量**. +- [ ] **国际化**. 支持中文. +- [ ] **前端易用性优化**. 更多高级功能,例如表格形式保存。 +- [ ] **日志自动清理**. +- [ ] **跟多文档**. + +## 未来计划 + +下一个版本有可能是 v0.6.0 的正式版本,但是没有确定。本次发布版本将会进行更多的测试,以保证其健壮性以及生产可用。 + diff --git a/changelog/v0.6.0-beta.20211116.md b/changelog/v0.6.0-beta.20211116.md new file mode 100644 index 00000000..f55e18f5 --- /dev/null +++ b/changelog/v0.6.0-beta.20211116.md @@ -0,0 +1,33 @@ +# Change Log (v0.6.0-beta.20211116) + +## Overview + +This is the second beta release for the next major version v0.6.0 after the [first beta release](https://github.com/crawlab-team/crawlab/releases/tag/v0.6.0-beta.20210803). With more features and optimization coming in, the release of official version v0.6.0 is approaching soon. + +## Enhancement + +#### Backend + +- [x] **Plugin Framework**. **Crawlab Plugin Framework (CPF)** has been released. See more info [here](https://docs-next.crawlab.cn/en/guide/use-crawlab/basic-concepts/plugin.html). +- [x] **Git Integration**. Git integration is implemented as a built-in feature. +- [x] **Scrapy Integration**. Scrapy integration is implemented as a plugin [spider-assistant](https://github.com/crawlab-team/plugin-spider-assistant). +- [x] **Dependency Integration**. Dependency integration is implemented as a plugin [dependency](https://github.com/crawlab-team/plugin-dependency). +- [x] **Notifications**. Notifications feature is implemented as a plugin [notification](https://github.com/crawlab-team/plugin-notification). +- [x] **Documentation Site**. Set up [documentation site](https://docs-next.crawlab.cn/en). + +#### Frontend +- **Bug Fixing**. + +#### TODOs +- [ ] **Associated Tasks**. There will be main tasks and their sub-tasks if task mode is "all nodes" or "selected nodes". +- [ ] **Crontab Editor**. Frontend component that visualize the crontab editing. +- [ ] **Results Deduplication**. +- [ ] **Environment Variables**. +- [ ] **Internationalization**. Support Chinese. +- [ ] **Frontend Utility Enhancement**. Advanced features such as saved table customization. +- [ ] **Log Auto Cleanup**. +- [ ] **More Documentation**. + +## What Next + +The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robostness and production-ready deployment. diff --git a/frontend/.browserslistrc b/frontend/.browserslistrc deleted file mode 100644 index 214388fe..00000000 --- a/frontend/.browserslistrc +++ /dev/null @@ -1,3 +0,0 @@ -> 1% -last 2 versions -not dead diff --git a/frontend/.dockerignore b/frontend/.dockerignore deleted file mode 100644 index c2658d7d..00000000 --- a/frontend/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -node_modules/ diff --git a/frontend/.editorconfig b/frontend/.editorconfig deleted file mode 100644 index 6fb90ce6..00000000 --- a/frontend/.editorconfig +++ /dev/null @@ -1,32 +0,0 @@ -root = true - -[*] -# charset = utf-8 -# end_of_line = lf -# indent_size = 4 -# indent_style = space -# insert_final_newline = true -# max_line_length = 120 -# tab_width = 4 -# trim_trailing_whitespace = true - -[*.scss] -# indent_size = 2 - -[{*.ats, *.ts}] -# indent_size = 2 -# tab_width = 2 - -[{*.js, *.cjs}] -# indent_size = 2 -# tab_width = 2 - -[{*.sht, *.html, *.shtm, *.shtml, *.htm, *.ng}] -# indent_size = 2 -# tab_width = 2 - -[{.analysis_options, *.yml, *.yaml}] -# indent_size = 2 - -[{.babelrc, .prettierrc, .stylelintrc, .eslintrc, jest.config, *.json, *.jsb3, *.jsb2, *.bowerrc}] -# indent_size = 2 diff --git a/frontend/.env b/frontend/.env new file mode 100644 index 00000000..28381cc1 --- /dev/null +++ b/frontend/.env @@ -0,0 +1 @@ +VUE_APP_API_BASE_URL= diff --git a/frontend/.env.development b/frontend/.env.development index 410388e1..3d2c6fb9 100644 --- a/frontend/.env.development +++ b/frontend/.env.development @@ -1,2 +1 @@ -NODE_ENV='development' VUE_APP_API_BASE_URL=http://localhost:8000 diff --git a/frontend/.env.docker b/frontend/.env.docker index 672216dd..63f2689b 100644 --- a/frontend/.env.docker +++ b/frontend/.env.docker @@ -1,2 +1 @@ -NODE_ENV='production' -VUE_APP_API_BASE_URL='VUE_APP_API_BASE_URL' +VUE_APP_API_BASE_URL=/api diff --git a/frontend/.env.production b/frontend/.env.production deleted file mode 100644 index d4c198ac..00000000 --- a/frontend/.env.production +++ /dev/null @@ -1,2 +0,0 @@ -NODE_ENV='production' -VUE_APP_API_BASE_URL=http://localhost:8000 diff --git a/frontend/.env.test b/frontend/.env.test deleted file mode 100644 index 12bf1f63..00000000 --- a/frontend/.env.test +++ /dev/null @@ -1,2 +0,0 @@ -NODE_ENV='test' -VUE_APP_API_BASE_URL=http://localhost:8000 diff --git a/frontend/.eslintignore b/frontend/.eslintignore index e960a9eb..b16742a5 100644 --- a/frontend/.eslintignore +++ b/frontend/.eslintignore @@ -1,2 +1,3 @@ -*/**/*.js ./src/i18n/**/*.ts +*/**/*.js +*.js diff --git a/frontend/.gitignore b/frontend/.gitignore index af135c7b..17b1b6fe 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -23,3 +23,9 @@ pnpm-debug.log* *.sw? tmp/ +lib/ +**/.DS_Store +**/.idea +**/dist +**/node_modules +**/package-lock.json diff --git a/frontend/LICENSE b/frontend/LICENSE deleted file mode 100644 index e41311d2..00000000 --- a/frontend/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2020, Crawlab Team -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/frontend/README.md b/frontend/README.md deleted file mode 100644 index 78e7a289..00000000 --- a/frontend/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# crawlab-frontend - -## Project setup -``` -yarn install -``` - -### Compiles and hot-reloads for development -``` -yarn serve -``` - -### Compiles and minifies for production -``` -yarn build -``` - -### Lints and fixes files -``` -yarn lint -``` - -### Customize configuration -See [Configuration Reference](https://cli.vuejs.org/config/). diff --git a/frontend/package.json b/frontend/package.json index d078d394..c4165cd8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,72 +1,27 @@ { - "name": "crawlab-frontend", - "version": "0.6.0-beta.20210715", - "private": false, + "name": "@crawlab/app", + "version": "0.6.0-beta.202111052150", + "description": "", "scripts": { - "serve": "vue-cli-service serve --port=8081", - "serve:build:local": "vue-cli-service serve --port=8082 --model local", + "serve": "vue-cli-service serve", + "serve:dist": "serve dist", "build": "vue-cli-service build", - "build:docker": "vue-cli-service build --mode docker", - "build:local": "vue-cli-service build --mode local", - "lint": "vue-cli-service lint", - "test": "jest" + "build:development": "vue-cli-service build --mode development", + "build:docker": "vue-cli-service build --mode docker" }, + "author": { + "name": "Marvin Zhang", + "email": "tikazyq@163.com" + }, + "license": "BSD-3-Clause", "dependencies": { - "@fortawesome/fontawesome-svg-core": "^1.2.32", - "@fortawesome/free-brands-svg-icons": "^5.15.1", - "@fortawesome/free-regular-svg-icons": "^5.15.1", - "@fortawesome/free-solid-svg-icons": "^5.15.1", - "@fortawesome/vue-fontawesome": "^3.0.0-2", - "@popperjs/core": "^2.6.0", - "@types/codemirror": "^0.0.103", - "@types/echarts": "^4.9.8", - "@types/getos": "^3.0.1", - "@types/humanize-duration": "^3.25.0", - "@types/javascript-time-ago": "^2.0.2", - "@types/md5": "^2.2.1", - "@types/pinyin": "^2.8.2", - "atom-material-icons": "^3.0.0", - "axios": "^0.21.1", - "codemirror": "^5.59.1", - "core-js": "^3.6.5", - "cron-parser": "^3.5.0", - "cronstrue": "^1.114.0", - "dayjs": "^1.10.5", - "echarts": "^5.1.2", - "element-plus": "1.0.2-beta.40", - "font-awesome": "^4.7.0", - "getos": "^3.2.1", - "humanize-duration": "^3.26.0", - "javascript-time-ago": "^2.3.6", - "md5": "^2.3.0", - "node-sass": "^5.0.0", - "normalize.css": "^8.0.1", - "pinyin": "^2.10.2", - "point-cluster": "^3.1.8", - "vue": "^3.0.4", - "vue-clipboard3": "^1.0.1", - "vue-i18n": "^9.0.0-beta.11", - "vue-router": "^4.0.0-0", - "vue3-dropzone": "^0.0.7", - "vuex": "^4.0.0-0" + "crawlab-ui": "0.6.0-beta.20211113.2052", + "vue": "3.0.11", + "vue-router": "^4.0.11" }, "devDependencies": { - "@babel/preset-typescript": "^7.12.7", - "@types/jest": "^26.0.19", - "@typescript-eslint/eslint-plugin": "^2.33.0", - "@typescript-eslint/parser": "^2.33.0", - "@vue/cli-plugin-babel": "~4.5.0", - "@vue/cli-plugin-eslint": "~4.5.0", - "@vue/cli-plugin-router": "~4.5.0", - "@vue/cli-plugin-typescript": "~4.5.0", - "@vue/cli-plugin-vuex": "~4.5.0", - "@vue/cli-service": "~4.5.0", - "@vue/compiler-sfc": "^3.0.0", - "@vue/eslint-config-typescript": "^5.0.2", - "eslint": "^6.7.2", - "eslint-plugin-vue": "^7.0.0-0", - "sass-loader": "^10.1.0", - "scss-loader": "^0.0.1", - "typescript": "~3.9.3" + "@vue/cli-service": "^4.5.13", + "@vue/compiler-sfc": "^3.0.11", + "serve": "^13.0.2" } } diff --git a/frontend/public/fonts/FontAwesome.otf b/frontend/public/fonts/FontAwesome.otf new file mode 100644 index 00000000..401ec0f3 Binary files /dev/null and b/frontend/public/fonts/FontAwesome.otf differ diff --git a/frontend/public/fonts/fontawesome-webfont.eot b/frontend/public/fonts/fontawesome-webfont.eot new file mode 100644 index 00000000..e9f60ca9 Binary files /dev/null and b/frontend/public/fonts/fontawesome-webfont.eot differ diff --git a/frontend/public/fonts/fontawesome-webfont.svg b/frontend/public/fonts/fontawesome-webfont.svg new file mode 100644 index 00000000..855c845e --- /dev/null +++ b/frontend/public/fonts/fontawesome-webfont.svg @@ -0,0 +1,2671 @@ + + + diff --git a/frontend/public/fonts/fontawesome-webfont.ttf b/frontend/public/fonts/fontawesome-webfont.ttf new file mode 100644 index 00000000..35acda2f Binary files /dev/null and b/frontend/public/fonts/fontawesome-webfont.ttf differ diff --git a/frontend/public/fonts/fontawesome-webfont.woff b/frontend/public/fonts/fontawesome-webfont.woff new file mode 100644 index 00000000..400014a4 Binary files /dev/null and b/frontend/public/fonts/fontawesome-webfont.woff differ diff --git a/frontend/public/fonts/fontawesome-webfont.woff2 b/frontend/public/fonts/fontawesome-webfont.woff2 new file mode 100644 index 00000000..4d13fc60 Binary files /dev/null and b/frontend/public/fonts/fontawesome-webfont.woff2 differ diff --git a/frontend/public/index.html b/frontend/public/index.html index 8147c22f..e71e5256 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -1,213 +1,214 @@
- - - - - -