From 51b31e56aa5ab75f2ae8c6dd35a7a63762e66829 Mon Sep 17 00:00:00 2001 From: hantmac Date: Thu, 19 Dec 2019 21:16:25 +0800 Subject: [PATCH 01/42] Add API GetVersion --- backend/conf/config.yml | 1 + backend/main.go | 2 ++ backend/routes/version.go | 17 +++++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 backend/routes/version.go diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 60d2bd41..d5503878 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -32,3 +32,4 @@ task: workers: 4 other: tmppath: "/tmp" +version: v0.4.1 \ No newline at end of file diff --git a/backend/main.go b/backend/main.go index 0d7b7cc1..ace026cc 100644 --- a/backend/main.go +++ b/backend/main.go @@ -176,6 +176,8 @@ func main() { authGroup.POST("/users/:id", routes.PostUser) // 更改用户 authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 authGroup.GET("/me", routes.GetMe) // 获取自己账户 + //release版本 + authGroup.GET("/version", routes.GetVersion) //获取发布的版本 } } diff --git a/backend/routes/version.go b/backend/routes/version.go new file mode 100644 index 00000000..719732ad --- /dev/null +++ b/backend/routes/version.go @@ -0,0 +1,17 @@ +package routes + +import ( + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net/http" +) + +func GetVersion(c *gin.Context) { + version := viper.GetString("version") + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: version, + }) +} From be4932f5aaf77c4783bcd07b1effc66c6df8b75c Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Dec 2019 09:58:04 +0800 Subject: [PATCH 02/42] fixed https://github.com/crawlab-team/crawlab/issues/371 --- backend/conf/config.yml | 2 +- frontend/src/views/layout/components/Sidebar/index.vue | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index d5503878..9ff0fa31 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -32,4 +32,4 @@ task: workers: 4 other: tmppath: "/tmp" -version: v0.4.1 \ No newline at end of file +version: 0.4.1 \ No newline at end of file diff --git a/frontend/src/views/layout/components/Sidebar/index.vue b/frontend/src/views/layout/components/Sidebar/index.vue index 1367cadd..35b599b9 100644 --- a/frontend/src/views/layout/components/Sidebar/index.vue +++ b/frontend/src/views/layout/components/Sidebar/index.vue @@ -51,8 +51,12 @@ export default { }, data () { return { - version: '0.4.1' + version: '' } + }, + async created () { + const res = await this.$request.get('/version') + this.version = res.data.data } } From 527506ea65f5461a17682296f2a2a73795babc1f Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Dec 2019 11:35:41 +0800 Subject: [PATCH 03/42] =?UTF-8?q?=E5=BF=85=E9=A1=BB=E5=90=8C=E6=84=8F?= =?UTF-8?q?=E5=85=8D=E8=B4=A3=E5=A3=B0=E6=98=8E=E6=89=8D=E8=83=BD=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=88=AC=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/Common/CrawlConfirmDialog.vue | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index 42fc2b6f..d25bfaaa 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -29,10 +29,14 @@ + + + 我已阅读并同意 《免责声明》 所有内容 + @@ -59,7 +63,8 @@ export default { nodeIds: undefined, param: '', nodeList: [] - } + }, + isAllowDisclaimer: true } }, methods: { @@ -82,6 +87,9 @@ export default { this.$emit('close') this.$st.sendEv('爬虫确认', '确认运行', this.form.runType) }) + }, + onClickDisclaimer () { + this.$router.push('/disclaimer') } }, created () { @@ -105,4 +113,8 @@ export default { .crawl-confirm-dialog >>> .el-form .el-form-item { margin-bottom: 20px; } + + .crawl-confirm-dialog >>> .disclaimer-wrapper a { + color: #409eff; + } From f766bf7a22bf8800e02dc7996854499096bb968e Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Dec 2019 11:36:52 +0800 Subject: [PATCH 04/42] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=85=8D=E8=B4=A3?= =?UTF-8?q?=E5=A3=B0=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DISCLAIMER-zh.md | 2 +- DISCLAIMER.md | 2 +- frontend/src/views/doc/Disclaimer.vue | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DISCLAIMER-zh.md b/DISCLAIMER-zh.md index 6a333d0b..a329e4e9 100644 --- a/DISCLAIMER-zh.md +++ b/DISCLAIMER-zh.md @@ -1,6 +1,6 @@ # 免责声明 -本免责及隐私保护声明(以下简称“隐私声明”或“本声明”)适用于 Crawlab 开发组 (以下简称“开发组”)研发的系列软件(以下简称"Crawlab") 在您阅读本声明后若不同意此声明中的任何条款，或对本声明存在质疑，请立刻停止使用我们的软件。若您已经开始或正在使用 Crawlab，则表示您已阅读并同意本声明的所有条款之约定。 +本免责及隐私保护声明(以下简称“免责声明”或“本声明”)适用于 Crawlab 开发组 (以下简称“开发组”)研发的系列软件(以下简称"Crawlab") 在您阅读本声明后若不同意此声明中的任何条款，或对本声明存在质疑，请立刻停止使用我们的软件。若您已经开始或正在使用 Crawlab，则表示您已阅读并同意本声明的所有条款之约定。 1. 总则：您通过安装 Crawlab 并使用 Crawlab 提供的服务与功能即表示您已经同意与开发组立本协议。开发组可随时执行全权决定更改“条款”。经修订的“条款”一经在 Github 免责声明页面上公布后，立即自动生效。 2. 本产品是基于Golang的分布式爬虫管理平台，支持Python、NodeJS、Go、Java、PHP等多种编程语言以及多种爬虫框架。 diff --git a/DISCLAIMER.md b/DISCLAIMER.md index 8f31739d..72aae961 100644 --- a/DISCLAIMER.md +++ b/DISCLAIMER.md @@ -1,6 +1,6 @@ # Disclaimer -This Disclaimer and privacy protection statement (hereinafter referred to as "privacy statement" or "this statement") is applicable to the series of software (hereinafter referred to as "crawlab") developed by crawlab development group (hereinafter referred to as "development group") after you read this statement, if you do not agree with any terms in this statement or have doubts about this statement, please stop using our software immediately. If you have started or are using crawlab, you have read and agree to all terms of this statement. +This Disclaimer and privacy protection statement (hereinafter referred to as "disclaimer statement" or "this statement") is applicable to the series of software (hereinafter referred to as "crawlab") developed by crawlab development group (hereinafter referred to as "development group") after you read this statement, if you do not agree with any terms in this statement or have doubts about this statement, please stop using our software immediately. If you have started or are using crawlab, you have read and agree to all terms of this statement. 1. General: by installing crawlab and using the services and functions provided by crawlab, you have agreed to establish this agreement with the development team. The developer group may at any time change the terms at its sole discretion. The amended "terms" shall take effect automatically as soon as they are published on the GitHub disclaimer page. 2. This product is a distributed crawler management platform based on golang, supporting python, nodejs, go, Java, PHP and other programming languages as well as a variety of crawler frameworks. diff --git a/frontend/src/views/doc/Disclaimer.vue b/frontend/src/views/doc/Disclaimer.vue index f9894942..07a55457 100644 --- a/frontend/src/views/doc/Disclaimer.vue +++ b/frontend/src/views/doc/Disclaimer.vue @@ -35,7 +35,7 @@ export default { textEn: ` # Disclaimer -This Disclaimer and privacy protection statement (hereinafter referred to as "privacy statement" or "this statement") is applicable to the series of software (hereinafter referred to as "crawlab") developed by crawlab development group (hereinafter referred to as "development group") after you read this statement, if you do not agree with any terms in this statement or have doubts about this statement, please stop using our software immediately. If you have started or are using crawlab, you have read and agree to all terms of this statement. +This Disclaimer and privacy protection statement (hereinafter referred to as "disclaimer statement" or "this statement") is applicable to the series of software (hereinafter referred to as "crawlab") developed by crawlab development group (hereinafter referred to as "development group") after you read this statement, if you do not agree with any terms in this statement or have doubts about this statement, please stop using our software immediately. If you have started or are using crawlab, you have read and agree to all terms of this statement. 1. General: by installing crawlab and using the services and functions provided by crawlab, you have agreed to establish this agreement with the development team. The developer group may at any time change the terms at its sole discretion. The amended "terms" shall take effect automatically as soon as they are published on the GitHub disclaimer page. 2. This product is a distributed crawler management platform based on golang, supporting python, nodejs, go, Java, PHP and other programming languages as well as a variety of crawler frameworks. @@ -49,7 +49,7 @@ This Disclaimer and privacy protection statement (hereinafter referred to as "pr textZh: ` # 免责声明 -本免责及隐私保护声明(以下简称“隐私声明”或“本声明”)适用于 Crawlab 开发组 (以下简称“开发组”)研发的系列软件(以下简称"Crawlab") 在您阅读本声明后若不同意此声明中的任何条款，或对本声明存在质疑，请立刻停止使用我们的软件。若您已经开始或正在使用 Crawlab，则表示您已阅读并同意本声明的所有条款之约定。 +本免责及隐私保护声明(下简称“免责声明”或“本声明”)适用于 Crawlab 开发组 (以下简称“开发组”)研发的系列软件(以下简称"Crawlab") 在您阅读本声明后若不同意此声明中的任何条款，或对本声明存在质疑，请立刻停止使用我们的软件。若您已经开始或正在使用 Crawlab，则表示您已阅读并同意本声明的所有条款之约定。 1. 总则：您通过安装 Crawlab 并使用 Crawlab 提供的服务与功能即表示您已经同意与开发组立本协议。开发组可随时执行全权决定更改“条款”。经修订的“条款”一经在 Github 免责声明页面上公布后，立即自动生效。 2. 本产品是基于Golang的分布式爬虫管理平台，支持Python、NodeJS、Go、Java、PHP等多种编程语言以及多种爬虫框架。 From 2132f423ebd694fd6185f609256e59ecff062b79 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Dec 2019 11:37:00 +0800 Subject: [PATCH 05/42] code cleanup --- frontend/src/views/doc/Disclaimer.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/views/doc/Disclaimer.vue b/frontend/src/views/doc/Disclaimer.vue index 07a55457..3c7f4c78 100644 --- a/frontend/src/views/doc/Disclaimer.vue +++ b/frontend/src/views/doc/Disclaimer.vue @@ -29,7 +29,6 @@ export default { }, data () { const converter = new showdown.Converter() - console.log(converter) return { converter, textEn: ` From 1a8fccc7786722310e1afc602b85d7c1634f6f3d Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Dec 2019 12:20:58 +0800 Subject: [PATCH 06/42] updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0973aa8a..3ba94ed6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # 0.4.2 (unknown) ### Features / Enhancement - **Disclaimer**. Added page for Disclaimer. +- **Call API to fetch version**. [#371](https://github.com/crawlab-team/crawlab/issues/371) # 0.4.1 (2019-12-13) ### Features / Enhancement From 9ffc79028a4ad9a31ead5d242406890d04ff9879 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Dec 2019 12:22:44 +0800 Subject: [PATCH 07/42] updated CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ba94ed6..b6745f71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ - **Disclaimer**. Added page for Disclaimer. - **Call API to fetch version**. [#371](https://github.com/crawlab-team/crawlab/issues/371) +### Bug Fixes +- **"mongodb no reachable" error**. [#373](https://github.com/crawlab-team/crawlab/issues/373) + # 0.4.1 (2019-12-13) ### Features / Enhancement - **Spiderfile Optimization**. Stages changed from dictionary to array. [#358](https://github.com/crawlab-team/crawlab/issues/358) From 28a3a53176fcc8b6cedcbe5718244b0823614563 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Dec 2019 12:45:37 +0800 Subject: [PATCH 08/42] fixed https://github.com/crawlab-team/crawlab/issues/346 --- backend/conf/config.yml | 4 +++- backend/main.go | 10 ++++---- backend/routes/setting.go | 33 +++++++++++++++++++++++++++ backend/routes/version.go | 17 -------------- frontend/src/App.vue | 7 ++++++ frontend/src/store/index.js | 2 ++ frontend/src/store/modules/setting.js | 28 +++++++++++++++++++++++ frontend/src/views/login/index.vue | 5 +++- 8 files changed, 82 insertions(+), 24 deletions(-) create mode 100644 backend/routes/setting.go delete mode 100644 backend/routes/version.go create mode 100644 frontend/src/store/modules/setting.js diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 9ff0fa31..a6522ba5 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -32,4 +32,6 @@ task: workers: 4 other: tmppath: "/tmp" -version: 0.4.1 \ No newline at end of file +version: 0.4.1 +setting: + allowRegister: "N" \ No newline at end of file diff --git a/backend/main.go b/backend/main.go index ace026cc..d14f64b7 100644 --- a/backend/main.go +++ b/backend/main.go @@ -114,9 +114,9 @@ func main() { app.Use(middlewares.CORSMiddleware()) anonymousGroup := app.Group("/") { - anonymousGroup.POST("/login", routes.Login) // 用户登录 - anonymousGroup.PUT("/users", routes.PutUser) // 添加用户 - + anonymousGroup.POST("/login", routes.Login) // 用户登录 + anonymousGroup.PUT("/users", routes.PutUser) // 添加用户 + anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息 } authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) { @@ -176,8 +176,8 @@ func main() { authGroup.POST("/users/:id", routes.PostUser) // 更改用户 authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 authGroup.GET("/me", routes.GetMe) // 获取自己账户 - //release版本 - authGroup.GET("/version", routes.GetVersion) //获取发布的版本 + // release版本 + authGroup.GET("/version", routes.GetVersion) // 获取发布的版本 } } diff --git a/backend/routes/setting.go b/backend/routes/setting.go new file mode 100644 index 00000000..4429873e --- /dev/null +++ b/backend/routes/setting.go @@ -0,0 +1,33 @@ +package routes + +import ( + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net/http" +) + +type SettingBody struct { + AllowRegister string `json:"allow_register"` +} + +func GetVersion(c *gin.Context) { + version := viper.GetString("version") + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: version, + }) +} + +func GetSetting(c *gin.Context) { + allowRegister := viper.GetString("setting.allowRegister") + + body := SettingBody{AllowRegister: allowRegister} + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: body, + }) +} diff --git a/backend/routes/version.go b/backend/routes/version.go deleted file mode 100644 index 719732ad..00000000 --- a/backend/routes/version.go +++ /dev/null @@ -1,17 +0,0 @@ -package routes - -import ( - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net/http" -) - -func GetVersion(c *gin.Context) { - version := viper.GetString("version") - - c.JSON(http.StatusOK, Response{ - Status: "ok", - Message: "success", - Data: version, - }) -} diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 2a91e61a..a7ba8069 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -6,6 +6,9 @@