From 0d612e92cf7f03be29000c3d37f89307e2ba0023 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 12 Mar 2020 09:21:55 +0800 Subject: [PATCH 01/42] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=A3=80=E6=B5=8Bscrap?= =?UTF-8?q?y=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 1 + backend/services/spider_handler/spider.go | 14 ++++++++------ docker-compose.yml | 3 ++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 6affb507..8c07a507 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -44,6 +44,7 @@ setting: enableTutorial: "N" runOnMaster: "Y" demoSpiders: "N" + checkScrapy: "Y" notification: mail: server: '' diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go index ab1d5774..d249c27b 100644 --- a/backend/services/spider_handler/spider.go +++ b/backend/services/spider_handler/spider.go @@ -45,13 +45,15 @@ func (s *SpiderSync) CheckIsScrapy() { if s.Spider.Type == constants.Configurable { return } + if viper.GetString("setting.checkScrapy") != "Y" { + return + } s.Spider.IsScrapy = utils.Exists(path.Join(s.Spider.Src, "scrapy.cfg")) - // TODO: 暂时停用自动检测Scrapy项目功能 - //if err := s.Spider.Save(); err != nil { - // log.Errorf(err.Error()) - // debug.PrintStack() - // return - //} + if err := s.Spider.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } } func (s *SpiderSync) AfterRemoveDownCreate() { diff --git a/docker-compose.yml b/docker-compose.yml index e9cb08cf..7f4f95da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,9 +19,9 @@ services: # CRAWLAB_LOG_LEVEL: "info" # log level 日志级别. 默认为 info # CRAWLAB_LOG_ISDELETEPERIODICALLY: "N" # whether to periodically delete log files 是否周期性删除日志文件. 默认不删除 # CRAWLAB_LOG_DELETEFREQUENCY: "@hourly" # frequency of deleting log files 删除日志文件的频率. 默认为每小时 + # CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数（并行执行任务数） # CRAWLAB_SERVER_REGISTER_TYPE: "mac" # node register type 节点注册方式. 默认为 mac 地址，也可设置为 ip（防止 mac 地址冲突） # CRAWLAB_SERVER_REGISTER_IP: "127.0.0.1" # node register ip 节点注册IP. 节点唯一识别号，只有当 CRAWLAB_SERVER_REGISTER_TYPE 为 "ip" 时才生效 - # CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数（并行执行任务数） # CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境 # CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境 # CRAWLAB_SERVER_LANG_DOTNET: "Y" # whether to pre-install .Net core 预安装 .Net Core 语言环境 @@ -30,6 +30,7 @@ services: # CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程 # CRAWLAB_SETTING_RUNONMASTER: "N" # whether to run on master node 是否在主节点上运行任务 # CRAWLAB_SETTING_DEMOSPIDERS: "Y" # whether to init demo spiders 是否使用Demo爬虫 + # CRAWLAB_SETTING_CHECKSCRAPY: "Y" # whether to automatically check if the spider is scrapy 是否自动检测爬虫为scrapy # CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址 # CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口 # CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱 From b41a28791584e9c8230c65d89679053796b7baa2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 12 Mar 2020 12:15:08 +0800 Subject: [PATCH 02/42] fixed https://github.com/crawlab-team/crawlab/issues/619 --- frontend/src/views/schedule/ScheduleList.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 68f8b012..3826f9db 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -582,6 +582,9 @@ export default { async onSpiderChange (spiderId) { await this.$store.dispatch('spider/getSpiderData', spiderId) if (this.spiderForm.type === 'customized' && this.spiderForm.is_scrapy) { + this.isLoading = true + await this.$store.dispatch('spider/getSpiderScrapySpiders', spiderId) + this.isLoading = false this.$set(this.scheduleForm, 'scrapy_spider', this.spiderForm.spider_names[0]) this.$set(this.scheduleForm, 'scrapy_log_level', 'INFO') } From 17b34b72da80e7b8fc45fc7b4878af5bb4f4f97f Mon Sep 17 00:00:00 2001 From: zkqiang Date: Wed, 18 Mar 2020 10:27:38 +0800 Subject: [PATCH 03/42] fixed GetNodeBaseInfo --- backend/model/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/model/node.go b/backend/model/node.go index 6e29a073..4d299f51 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -266,7 +266,7 @@ func GetNodeBaseInfo() (ip string, mac string, hostname string, key string, erro debug.PrintStack() return "", "", "", "", err } - return ip, mac, key, hostname, nil + return ip, mac, hostname, key, nil } // 根据redis的key值，重置node节点为offline From e0d953d18f2717f7a5766f32665701a1ae784c7a Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 18 Mar 2020 18:00:25 +0800 Subject: [PATCH 04/42] =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=8C=91=E6=88=98?= =?UTF-8?q?=E5=9F=BA=E7=A1=80=E5=87=86=E5=A4=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG-zh.md | 6 + CHANGELOG.md | 6 + backend/constants/action.go | 5 + backend/constants/challenge.go | 7 + backend/data/challenge_data.json | 28 +++ backend/entity/rpc.go | 14 +- backend/main.go | 22 +- backend/model/action.go | 115 +++++++++++ backend/model/challenge.go | 154 ++++++++++++++ backend/routes/action.go | 114 +++++++++++ backend/routes/challenge.go | 31 +++ backend/services/challenge/base.go | 101 ++++++++++ .../challenge/create_customized_spider.go | 8 + backend/services/challenge/login_7d.go | 8 + backend/services/challenge/run_random.go | 8 + frontend/src/App.vue | 5 + frontend/src/i18n/zh.js | 8 + frontend/src/router/index.js | 23 ++- .../src/views/challenge/ChallengeList.vue | 188 ++++++++++++++++++ 19 files changed, 841 insertions(+), 10 deletions(-) create mode 100644 backend/constants/action.go create mode 100644 backend/constants/challenge.go create mode 100644 backend/data/challenge_data.json create mode 100644 backend/model/action.go create mode 100644 backend/model/challenge.go create mode 100644 backend/routes/action.go create mode 100644 backend/routes/challenge.go create mode 100644 backend/services/challenge/base.go create mode 100644 backend/services/challenge/create_customized_spider.go create mode 100644 backend/services/challenge/login_7d.go create mode 100644 backend/services/challenge/run_random.go create mode 100644 frontend/src/views/challenge/ChallengeList.vue diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index 4099e43a..02c903ce 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,3 +1,9 @@ +# 0.4.9 (TBC) +### 功能 / 优化 + +### Bug 修复 +- **CLI 无法在 Windows 上使用**. [#580](https://github.com/crawlab-team/crawlab/issues/580) + # 0.4.8 (2020-03-11) ### 功能 / 优化 - **支持更多编程语言安装**. 现在用户可以安装或预装更多的编程语言，包括 Java、.Net Core、PHP. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b4792f3..d5e7c72e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 0.4.9 (TBC) +### Features / Enhancement + +### Bug Fixes +- **CLI unable to use on Windows**. [#580](https://github.com/crawlab-team/crawlab/issues/580) + # 0.4.8 (2020-03-11) ### Features / Enhancement - **Support Installations of More Programming Languages**. Now users can install or pre-install more programming languages including Java, .Net Core and PHP. diff --git a/backend/constants/action.go b/backend/constants/action.go new file mode 100644 index 00000000..31d3655e --- /dev/null +++ b/backend/constants/action.go @@ -0,0 +1,5 @@ +package constants + +const ( + ActionTypeVisit = "visit" +) diff --git a/backend/constants/challenge.go b/backend/constants/challenge.go new file mode 100644 index 00000000..4cfa9710 --- /dev/null +++ b/backend/constants/challenge.go @@ -0,0 +1,7 @@ +package constants + +const ( + ChallengeLogin7d = "login_7d" + ChallengeCreateCustomizedSpider = "create_customized_spider" + ChallengeRunRandom = "run_random" +) diff --git a/backend/data/challenge_data.json b/backend/data/challenge_data.json new file mode 100644 index 00000000..e8cea7da --- /dev/null +++ b/backend/data/challenge_data.json @@ -0,0 +1,28 @@ +[ + { + "name": "login_7d", + "title_cn": "连续登录 7 天", + "title_en": "Logged-in for 7 days", + "description_cn": "连续 7 天登录 Crawlab，即可完成挑战！", + "description_en": "Logged-in for consecutive 7 days to complete the challenge", + "difficulty": 1 + }, + { + "name": "create_customized_spider", + "title_cn": "创建一个自定义爬虫", + "title_en": "Create a customized spider", + "description_cn": "在爬虫列表中，点击 '添加爬虫'，选择 '自定义爬虫'，输入相应的参数，点击添加，即可完成挑战！", + "description_en": "In Spider List page, click 'Add Spider', select 'Customized Spider', enter params, click 'Add' to finish the challenge.", + "difficulty": 1, + "path": "/spiders" + }, + { + "name": "run_random", + "title_cn": "用随机模式运行一个爬虫", + "title_en": "Run a spider in random mode", + "description_cn": "在您创建好的爬虫中，导航到其对应的详情页（爬虫列表中点击爬虫），运行一个爬虫，选择随机模式。", + "description_en": "In your created spiders, navigate to corresponding detail page (click spider in Spider List page), run a spider in random mode.", + "difficulty": 2, + "path": "/spiders" + } +] \ No newline at end of file diff --git a/backend/entity/rpc.go b/backend/entity/rpc.go index 3f5ddcea..48f14b26 100644 --- a/backend/entity/rpc.go +++ b/backend/entity/rpc.go @@ -1,11 +1,11 @@ package entity type RpcMessage struct { - Id string `json:"id"` - Method string `json:"method"` - NodeId string `json:"node_id"` - Params map[string]string `json:"params"` - Timeout int `json:"timeout"` - Result string `json:"result"` - Error string `json:"error"` + Id string `json:"id"` // 消息ID + Method string `json:"method"` // 消息方法 + NodeId string `json:"node_id"` // 节点ID + Params map[string]string `json:"params"` // 参数 + Timeout int `json:"timeout"` // 超时 + Result string `json:"result"` // 结果 + Error string `json:"error"` // 错误 } diff --git a/backend/main.go b/backend/main.go index d53991e2..1b568ace 100644 --- a/backend/main.go +++ b/backend/main.go @@ -9,6 +9,7 @@ import ( "crawlab/model" "crawlab/routes" "crawlab/services" + "crawlab/services/challenge" "crawlab/services/rpc" "github.com/apex/log" "github.com/gin-gonic/gin" @@ -91,6 +92,14 @@ func main() { panic(err) } log.Info("initialized dependency fetcher successfully") + + // 初始化挑战服务 + if err := challenge.InitChallengeService(); err != nil { + log.Error("init challenge service error:" + err.Error()) + debug.PrintStack() + panic(err) + } + log.Info("initialized challenge service successfully") } // 初始化任务执行器 @@ -254,6 +263,17 @@ func main() { authGroup.POST("/projects/:id", routes.PostProject) // 新增 authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除 } + // 挑战 + { + authGroup.GET("/challenges", routes.GetChallengeList) // 挑战列表 + } + // 操作 + { + //authGroup.GET("/actions", routes.GetActionList) // 操作列表 + //authGroup.GET("/actions/:id", routes.GetAction) // 操作 + authGroup.PUT("/actions", routes.PutAction) // 新增操作 + //authGroup.POST("/actions/:id", routes.PostAction) // 修改操作 + } // 统计数据 authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 // 文件 @@ -262,7 +282,7 @@ func main() { authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支 authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits - authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits + authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits } } diff --git a/backend/model/action.go b/backend/model/action.go new file mode 100644 index 00000000..14968cec --- /dev/null +++ b/backend/model/action.go @@ -0,0 +1,115 @@ +package model + +import ( + "crawlab/database" + "github.com/apex/log" + "github.com/globalsign/mgo/bson" + "runtime/debug" + "time" +) + +type Action struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + Type string `json:"type" bson:"type"` + + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` +} + +func (a *Action) Save() error { + s, c := database.GetCol("actions") + defer s.Close() + + a.UpdateTs = time.Now() + + if err := c.UpdateId(a.Id, a); err != nil { + debug.PrintStack() + return err + } + return nil +} + +func (a *Action) Add() error { + s, c := database.GetCol("actions") + defer s.Close() + + a.Id = bson.NewObjectId() + a.UpdateTs = time.Now() + a.CreateTs = time.Now() + if err := c.Insert(a); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +func GetAction(id bson.ObjectId) (Action, error) { + s, c := database.GetCol("actions") + defer s.Close() + var user Action + if err := c.Find(bson.M{"_id": id}).One(&user); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return user, err + } + return user, nil +} + +func GetActionList(filter interface{}, skip int, limit int, sortKey string) ([]Action, error) { + s, c := database.GetCol("actions") + defer s.Close() + + var actions []Action + if err := c.Find(filter).Skip(skip).Limit(limit).Sort(sortKey).All(&actions); err != nil { + debug.PrintStack() + return actions, err + } + return actions, nil +} + +func GetActionListTotal(filter interface{}) (int, error) { + s, c := database.GetCol("actions") + defer s.Close() + + var result int + result, err := c.Find(filter).Count() + if err != nil { + return result, err + } + return result, nil +} + +func UpdateAction(id bson.ObjectId, item Action) error { + s, c := database.GetCol("actions") + defer s.Close() + + var result Action + if err := c.FindId(id).One(&result); err != nil { + debug.PrintStack() + return err + } + + if err := item.Save(); err != nil { + return err + } + return nil +} + +func RemoveAction(id bson.ObjectId) error { + s, c := database.GetCol("actions") + defer s.Close() + + var result Action + if err := c.FindId(id).One(&result); err != nil { + return err + } + + if err := c.RemoveId(id); err != nil { + return err + } + + return nil +} diff --git a/backend/model/challenge.go b/backend/model/challenge.go new file mode 100644 index 00000000..9a96660b --- /dev/null +++ b/backend/model/challenge.go @@ -0,0 +1,154 @@ +package model + +import ( + "crawlab/database" + "github.com/apex/log" + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" + "runtime/debug" + "time" +) + +type Challenge struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + TitleCn string `json:"title_cn" bson:"title_cn"` + TitleEn string `json:"title_en" bson:"title_en"` + DescriptionCn string `json:"description_cn" bson:"description_cn"` + DescriptionEn string `json:"description_en" bson:"description_en"` + Difficulty int `json:"difficulty" bson:"difficulty"` + Path string `json:"path" bson:"path"` + + // 前端展示 + Achieved bool `json:"achieved" bson:"achieved"` + + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` +} + +func (ch *Challenge) Save() error { + s, c := database.GetCol("challenges") + defer s.Close() + + ch.UpdateTs = time.Now() + + if err := c.UpdateId(ch.Id, c); err != nil { + debug.PrintStack() + return err + } + return nil +} + +func (ch *Challenge) Add() error { + s, c := database.GetCol("challenges") + defer s.Close() + + ch.Id = bson.NewObjectId() + ch.UpdateTs = time.Now() + ch.CreateTs = time.Now() + if err := c.Insert(ch); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +func GetChallenge(id bson.ObjectId) (Challenge, error) { + s, c := database.GetCol("challenges") + defer s.Close() + + var ch Challenge + if err := c.Find(bson.M{"_id": id}).One(&ch); err != nil { + if err != mgo.ErrNotFound { + log.Errorf(err.Error()) + debug.PrintStack() + return ch, err + } + } + + return ch, nil +} + +func GetChallengeByName(name string) (Challenge, error) { + s, c := database.GetCol("challenges") + defer s.Close() + + var ch Challenge + if err := c.Find(bson.M{"name": name}).One(&ch); err != nil { + if err != mgo.ErrNotFound { + log.Errorf(err.Error()) + debug.PrintStack() + return ch, err + } + } + + return ch, nil +} + +func GetChallengeList(filter interface{}, skip int, limit int, sortKey string) ([]Challenge, error) { + s, c := database.GetCol("challenges") + defer s.Close() + + var challenges []Challenge + if err := c.Find(filter).Skip(skip).Limit(limit).Sort(sortKey).All(&challenges); err != nil { + debug.PrintStack() + return challenges, err + } + + //for _, ch := range challenges { + //} + + return challenges, nil +} + +func GetChallengeListTotal(filter interface{}) (int, error) { + s, c := database.GetCol("challenges") + defer s.Close() + + var result int + result, err := c.Find(filter).Count() + if err != nil { + return result, err + } + return result, nil +} + +type ChallengeAchievement struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + ChallengeId bson.ObjectId `json:"challenge_id" bson:"challenge_id"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` +} + +func (ca *ChallengeAchievement) Save() error { + s, c := database.GetCol("challenges_achievements") + defer s.Close() + + ca.UpdateTs = time.Now() + + if err := c.UpdateId(ca.Id, c); err != nil { + debug.PrintStack() + return err + } + return nil +} + +func (ca *ChallengeAchievement) Add() error { + s, c := database.GetCol("challenges_achievements") + defer s.Close() + + ca.Id = bson.NewObjectId() + ca.UpdateTs = time.Now() + ca.CreateTs = time.Now() + if err := c.Insert(ca); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} diff --git a/backend/routes/action.go b/backend/routes/action.go new file mode 100644 index 00000000..26ffe66e --- /dev/null +++ b/backend/routes/action.go @@ -0,0 +1,114 @@ +package routes + +import ( + "crawlab/model" + "crawlab/services" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" +) + +func GetAction(c *gin.Context) { + id := c.Param("id") + + user, err := model.GetAction(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: user, + }) +} + +func GetActionList(c *gin.Context) { + pageNum := c.GetInt("page_num") + pageSize := c.GetInt("page_size") + + users, err := model.GetActionList(nil, (pageNum-1)*pageSize, pageSize, "-create_ts") + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + total, err := model.GetActionListTotal(nil) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, ListResponse{ + Status: "ok", + Message: "success", + Data: users, + Total: total, + }) +} + +func PutAction(c *gin.Context) { + // 绑定请求数据 + var action model.Action + if err := c.ShouldBindJSON(&action); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + action.UserId = services.GetCurrentUser(c).Id + + if err := action.Add(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func PostAction(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + } + + var item model.Action + if err := c.ShouldBindJSON(&item); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + if err := model.UpdateAction(bson.ObjectIdHex(id), item); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func DeleteAction(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } + + // 从数据库中删除该爬虫 + if err := model.RemoveAction(bson.ObjectIdHex(id)); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/routes/challenge.go b/backend/routes/challenge.go new file mode 100644 index 00000000..f8d09b6e --- /dev/null +++ b/backend/routes/challenge.go @@ -0,0 +1,31 @@ +package routes + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/gin-gonic/gin" + "net/http" +) + +func GetChallengeList(c *gin.Context) { + // 获取列表 + users, err := model.GetChallengeList(nil, 0, constants.Infinite, "create_ts") + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取总数 + total, err := model.GetChallengeListTotal(nil) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, ListResponse{ + Status: "ok", + Message: "success", + Data: users, + Total: total, + }) +} diff --git a/backend/services/challenge/base.go b/backend/services/challenge/base.go new file mode 100644 index 00000000..6529d887 --- /dev/null +++ b/backend/services/challenge/base.go @@ -0,0 +1,101 @@ +package challenge + +import ( + "crawlab/constants" + "crawlab/model" + "encoding/json" + "github.com/apex/log" + "github.com/globalsign/mgo/bson" + "io/ioutil" + "path" + "runtime/debug" +) + +type Service interface { + Check() (bool, error) +} + +func GetService(name string) Service { + switch name { + case constants.ChallengeLogin7d: + return &Login7dService{} + case constants.ChallengeCreateCustomizedSpider: + return &CreateCustomizedSpiderService{} + case constants.ChallengeRunRandom: + return &RunRandomService{} + } + return nil +} + +func AddChallengeAchievement(name string, uid bson.ObjectId) error { + ch, err := model.GetChallengeByName(name) + if err != nil { + return err + } + ca := model.ChallengeAchievement{ + ChallengeId: ch.Id, + UserId: uid, + } + if err := ca.Add(); err != nil { + return err + } + return nil +} + +func CheckChallengeAndUpdate(name string, uid bson.ObjectId) error { + svc := GetService(name) + achieved, err := svc.Check() + if err != nil { + return err + } + if achieved { + if err := AddChallengeAchievement(name, uid); err != nil { + return err + } + } + return nil +} + +func CheckChallengeAndUpdateAll(uid bson.ObjectId) error { + return nil +} + +func InitChallengeService() error { + // 读取文件 + contentBytes, err := ioutil.ReadFile(path.Join("data", "challenge_data.json")) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 反序列化 + var challenges []model.Challenge + if err := json.Unmarshal(contentBytes, &challenges); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + for _, ch := range challenges { + chDb, err := model.GetChallengeByName(ch.Name) + if err != nil { + continue + } + if chDb.Name == "" { + if err := ch.Add(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } else { + if err := ch.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } + } + + return nil +} diff --git a/backend/services/challenge/create_customized_spider.go b/backend/services/challenge/create_customized_spider.go new file mode 100644 index 00000000..cad1aa25 --- /dev/null +++ b/backend/services/challenge/create_customized_spider.go @@ -0,0 +1,8 @@ +package challenge + +type CreateCustomizedSpiderService struct { +} + +func (s *CreateCustomizedSpiderService) Check() (bool, error) { + return true, nil +} diff --git a/backend/services/challenge/login_7d.go b/backend/services/challenge/login_7d.go new file mode 100644 index 00000000..c10e2d38 --- /dev/null +++ b/backend/services/challenge/login_7d.go @@ -0,0 +1,8 @@ +package challenge + +type Login7dService struct { +} + +func (s *Login7dService) Check() (bool, error) { + return true, nil +} diff --git a/backend/services/challenge/run_random.go b/backend/services/challenge/run_random.go new file mode 100644 index 00000000..dd9e4a40 --- /dev/null +++ b/backend/services/challenge/run_random.go @@ -0,0 +1,8 @@ +package challenge + +type RunRandomService struct { +} + +func (s *RunRandomService) Check() (bool, error) { + return true, nil +} diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 93d41827..1917a490 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -51,6 +51,11 @@ export default { // remove loading-placeholder const elLoading = document.querySelector('#loading-placeholder') elLoading.remove() + + // send visit event + await this.$request.put('/actions', { + type: 'visit' + }) } } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 899d9d44..e677d90b 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -459,6 +459,14 @@ export default { 'General': '通用', 'Enable Tutorial': '启用教程', + // 挑战 + 'Challenge': '挑战', + 'Challenges': '挑战', + 'Difficulty': '难度', + 'Achieved': '已达成', + 'Not Achieved': '未达成', + 'Start Challenge': '开始挑战', + // 全局 'Related Documentation': '相关文档', 'Click to view related Documentation': '点击查看相关文档', diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 69b7f35b..276fe49d 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -197,7 +197,7 @@ export const constantRouterMap = [ component: Layout, meta: { title: 'User', - icon: 'fa fa-user' + icon: 'fa fa-users' }, children: [ { @@ -206,7 +206,26 @@ export const constantRouterMap = [ component: () => import('../views/user/UserList'), meta: { title: 'Users', - icon: 'fa fa-user' + icon: 'fa fa-users' + } + } + ] + }, + { + path: '/challenges', + component: Layout, + meta: { + title: 'User', + icon: 'fa fa-flash' + }, + children: [ + { + path: '', + name: 'ChallengeList', + component: () => import('../views/challenge/ChallengeList'), + meta: { + title: 'Challenges', + icon: 'fa fa-flash' } } ] diff --git a/frontend/src/views/challenge/ChallengeList.vue b/frontend/src/views/challenge/ChallengeList.vue new file mode 100644 index 00000000..c0bda6f1 --- /dev/null +++ b/frontend/src/views/challenge/ChallengeList.vue @@ -0,0 +1,188 @@ + + + + + From daef59be072b4a771237d1d6937bba4a310cda22 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 19 Mar 2020 11:13:25 +0800 Subject: [PATCH 05/42] updated contributors --- README-zh.md | 3 ++ README.md | 3 ++ backend/main.go | 3 +- backend/mock/spider.go | 2 + backend/mock/spider_test.go | 2 + backend/model/action.go | 47 +++++++++++++++++++ backend/model/challenge.go | 37 ++++++++++++++- backend/model/project.go | 7 +-- backend/model/spider.go | 5 +- backend/model/task.go | 8 ++-- backend/routes/action.go | 2 +- backend/routes/challenge.go | 16 ++++++- backend/routes/schedule.go | 2 +- backend/routes/spider.go | 17 +++++-- backend/routes/task.go | 9 ++-- backend/services/challenge/base.go | 25 ++++++---- .../challenge/create_customized_spider.go | 17 ++++++- backend/services/challenge/login_7d.go | 12 ++++- backend/services/challenge/run_random.go | 17 ++++++- backend/services/spider.go | 2 + backend/services/user.go | 5 ++ .../src/views/challenge/ChallengeList.vue | 1 + 22 files changed, 211 insertions(+), 31 deletions(-) diff --git a/README-zh.md b/README-zh.md index 1adf3dc7..fabab977 100644 --- a/README-zh.md +++ b/README-zh.md @@ -305,6 +305,9 @@ Crawlab使用起来很方便，也很通用，可以适用于几乎任何主流 + + + ## 社区 & 赞助 diff --git a/README.md b/README.md index 25a16c8e..edd3b6d7 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,9 @@ Crawlab is easy to use, general enough to adapt spiders in any language and any + + + ## Community & Sponsorship diff --git a/backend/main.go b/backend/main.go index 1b568ace..6550bbbd 100644 --- a/backend/main.go +++ b/backend/main.go @@ -265,7 +265,8 @@ func main() { } // 挑战 { - authGroup.GET("/challenges", routes.GetChallengeList) // 挑战列表 + authGroup.GET("/challenges", routes.GetChallengeList) // 挑战列表 + authGroup.POST("/challenges-check", routes.CheckChallengeList) // 检查挑战列表 } // 操作 { diff --git a/backend/mock/spider.go b/backend/mock/spider.go index ef3e6104..e6c20a6b 100644 --- a/backend/mock/spider.go +++ b/backend/mock/spider.go @@ -1,6 +1,7 @@ package mock import ( + "crawlab/constants" "crawlab/model" "github.com/apex/log" "github.com/gin-gonic/gin" @@ -26,6 +27,7 @@ var SpiderList = []model.Spider{ LastRunTs: time.Now(), CreateTs: time.Now(), UpdateTs: time.Now(), + UserId: constants.ObjectIdNull, }, } diff --git a/backend/mock/spider_test.go b/backend/mock/spider_test.go index f4dbea63..11e2c9cd 100644 --- a/backend/mock/spider_test.go +++ b/backend/mock/spider_test.go @@ -2,6 +2,7 @@ package mock import ( "bytes" + "crawlab/constants" "crawlab/model" "encoding/json" "github.com/globalsign/mgo/bson" @@ -61,6 +62,7 @@ func TestPostSpider(t *testing.T) { LastRunTs: time.Now(), CreateTs: time.Now(), UpdateTs: time.Now(), + UserId: constants.ObjectIdNull, } var resp Response var spiderId = "5d429e6c19f7abede924fee2" diff --git a/backend/model/action.go b/backend/model/action.go index 14968cec..15406181 100644 --- a/backend/model/action.go +++ b/backend/model/action.go @@ -1,6 +1,7 @@ package model import ( + "crawlab/constants" "crawlab/database" "github.com/apex/log" "github.com/globalsign/mgo/bson" @@ -82,6 +83,52 @@ func GetActionListTotal(filter interface{}) (int, error) { return result, nil } +func GetVisitDays(uid bson.ObjectId) (int, error) { + type ResData struct { + Days int `json:"days" bson:"days"` + } + s, c := database.GetCol("actions") + defer s.Close() + + pipeline := []bson.M{ + { + "$match": bson.M{ + "user_id": uid, + "type": constants.ActionTypeVisit, + }, + }, + { + "$addFields": bson.M{ + "date": bson.M{ + "$dateToString": bson.M{ + "format": "%Y%m%d", + "date": "$create_ts", + "timezone": "Asia/Shanghai", + }, + }, + }, + }, + { + "$group": bson.M{ + "_id": "$date", + }, + }, + { + "_id": nil, + "days": bson.M{"$sum": 1}, + }, + } + + var resData []ResData + if err := c.Pipe(pipeline).All(&resData); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return 0, err + } + + return resData[0].Days, nil +} + func UpdateAction(id bson.ObjectId, item Action) error { s, c := database.GetCol("actions") defer s.Close() diff --git a/backend/model/challenge.go b/backend/model/challenge.go index 9a96660b..083d216b 100644 --- a/backend/model/challenge.go +++ b/backend/model/challenge.go @@ -97,8 +97,28 @@ func GetChallengeList(filter interface{}, skip int, limit int, sortKey string) ( return challenges, err } - //for _, ch := range challenges { - //} + return challenges, nil +} + +func GetChallengeListWithAchieved(filter interface{}, skip int, limit int, sortKey string, uid bson.ObjectId) ([]Challenge, error) { + challenges, err := GetChallengeList(filter, skip, limit, sortKey) + if err != nil { + return challenges, err + } + + for i, ch := range challenges { + query := bson.M{ + "user_id": uid, + "challenge_id": ch.Id, + } + + list, err := GetChallengeAchievementList(query, 0, 1, "-_id") + if err != nil { + continue + } + + challenges[i].Achieved = len(list) > 0 + } return challenges, nil } @@ -152,3 +172,16 @@ func (ca *ChallengeAchievement) Add() error { return nil } + +func GetChallengeAchievementList(filter interface{}, skip int, limit int, sortKey string) ([]ChallengeAchievement, error) { + s, c := database.GetCol("challenges_achievements") + defer s.Close() + + var challengeAchievements []ChallengeAchievement + if err := c.Find(filter).Skip(skip).Limit(limit).Sort(sortKey).All(&challengeAchievements); err != nil { + debug.PrintStack() + return challengeAchievements, err + } + + return challengeAchievements, nil +} diff --git a/backend/model/project.go b/backend/model/project.go index 92c72655..bc7ca35e 100644 --- a/backend/model/project.go +++ b/backend/model/project.go @@ -15,11 +15,12 @@ type Project struct { Description string `json:"description" bson:"description"` Tags []string `json:"tags" bson:"tags"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` - // 前端展示 Spiders []Spider `json:"spiders" bson:"spiders"` + + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } func (p *Project) Save() error { diff --git a/backend/model/spider.go b/backend/model/spider.go index 49f735a4..5bc2579b 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -65,8 +65,9 @@ type Spider struct { LatestTasks []Task `json:"latest_tasks"` // 最近任务列表 // 时间 - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } // 更新爬虫 diff --git a/backend/model/task.go b/backend/model/task.go index abb5ffc5..6b2a44c1 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -25,14 +25,16 @@ type Task struct { RuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"` TotalDuration float64 `json:"total_duration" bson:"total_duration"` Pid int `json:"pid" bson:"pid"` - UserId bson.ObjectId `json:"user_id" bson:"user_id"` + RunType string `json:"run_type" bson:"run_type"` + ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"` // 前端数据 SpiderName string `json:"spider_name"` NodeName string `json:"node_name"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } type TaskDailyItem struct { diff --git a/backend/routes/action.go b/backend/routes/action.go index 26ffe66e..f0d262ab 100644 --- a/backend/routes/action.go +++ b/backend/routes/action.go @@ -56,7 +56,7 @@ func PutAction(c *gin.Context) { return } - action.UserId = services.GetCurrentUser(c).Id + action.UserId = services.GetCurrentUserId(c) if err := action.Add(); err != nil { HandleError(http.StatusInternalServerError, c, err) diff --git a/backend/routes/challenge.go b/backend/routes/challenge.go index f8d09b6e..1f03654a 100644 --- a/backend/routes/challenge.go +++ b/backend/routes/challenge.go @@ -3,13 +3,15 @@ package routes import ( "crawlab/constants" "crawlab/model" + "crawlab/services" + "crawlab/services/challenge" "github.com/gin-gonic/gin" "net/http" ) func GetChallengeList(c *gin.Context) { // 获取列表 - users, err := model.GetChallengeList(nil, 0, constants.Infinite, "create_ts") + users, err := model.GetChallengeListWithAchieved(nil, 0, constants.Infinite, "create_ts", services.GetCurrentUserId(c)) if err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -29,3 +31,15 @@ func GetChallengeList(c *gin.Context) { Total: total, }) } + +func CheckChallengeList(c *gin.Context) { + uid := services.GetCurrentUserId(c) + if err := challenge.CheckChallengeAndUpdateAll(uid); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index 3776019c..601d467f 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -77,7 +77,7 @@ func PutSchedule(c *gin.Context) { } // 加入用户ID - item.UserId = services.GetCurrentUser(c).Id + item.UserId = services.GetCurrentUserId(c) // 更新数据库 if err := model.AddSchedule(item); err != nil { diff --git a/backend/routes/spider.go b/backend/routes/spider.go index d195a5e7..b136070b 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -189,6 +189,9 @@ func PutSpider(c *gin.Context) { // 将FileId置空 spider.FileId = bson.ObjectIdHex(constants.ObjectIdNull) + // UserId + spider.UserId = services.GetCurrentUserId(c) + // 爬虫目录 spiderDir := filepath.Join(viper.GetString("spider.path"), spider.Name) @@ -274,6 +277,9 @@ func CopySpider(c *gin.Context) { return } + // UserId + spider.UserId = services.GetCurrentUserId(c) + // 复制爬虫 if err := services.CopySpider(spider, reqBody.Name); err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -365,6 +371,8 @@ func UploadSpider(c *gin.Context) { Type: constants.Customized, Src: filepath.Join(srcPath, spiderName), FileId: fid, + ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), + UserId: services.GetCurrentUserId(c), } if name != "" { spider.Name = name @@ -617,7 +625,8 @@ func RunSelectedSpider(c *gin.Context) { SpiderId: taskParam.SpiderId, NodeId: node.Id, Param: taskParam.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeAllNodes, } id, err := services.AddTask(t) @@ -633,7 +642,8 @@ func RunSelectedSpider(c *gin.Context) { t := model.Task{ SpiderId: taskParam.SpiderId, Param: taskParam.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeRandom, } id, err := services.AddTask(t) if err != nil { @@ -648,7 +658,8 @@ func RunSelectedSpider(c *gin.Context) { SpiderId: taskParam.SpiderId, NodeId: nodeId, Param: taskParam.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeSelectedNodes, } id, err := services.AddTask(t) diff --git a/backend/routes/task.go b/backend/routes/task.go index 2880abb9..ecfbc71f 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -115,7 +115,8 @@ func PutTask(c *gin.Context) { SpiderId: reqBody.SpiderId, NodeId: node.Id, Param: reqBody.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeAllNodes, } id, err := services.AddTask(t) @@ -131,7 +132,8 @@ func PutTask(c *gin.Context) { t := model.Task{ SpiderId: reqBody.SpiderId, Param: reqBody.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeRandom, } id, err := services.AddTask(t) if err != nil { @@ -146,7 +148,8 @@ func PutTask(c *gin.Context) { SpiderId: reqBody.SpiderId, NodeId: nodeId, Param: reqBody.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeSelectedNodes, } id, err := services.AddTask(t) diff --git a/backend/services/challenge/base.go b/backend/services/challenge/base.go index 6529d887..9515ff9e 100644 --- a/backend/services/challenge/base.go +++ b/backend/services/challenge/base.go @@ -15,14 +15,14 @@ type Service interface { Check() (bool, error) } -func GetService(name string) Service { +func GetService(name string, uid bson.ObjectId) Service { switch name { case constants.ChallengeLogin7d: - return &Login7dService{} + return &Login7dService{UserId: uid} case constants.ChallengeCreateCustomizedSpider: - return &CreateCustomizedSpiderService{} + return &CreateCustomizedSpiderService{UserId: uid} case constants.ChallengeRunRandom: - return &RunRandomService{} + return &RunRandomService{UserId: uid} } return nil } @@ -42,14 +42,14 @@ func AddChallengeAchievement(name string, uid bson.ObjectId) error { return nil } -func CheckChallengeAndUpdate(name string, uid bson.ObjectId) error { - svc := GetService(name) +func CheckChallengeAndUpdate(ch model.Challenge, uid bson.ObjectId) error { + svc := GetService(ch.Name, uid) achieved, err := svc.Check() if err != nil { return err } - if achieved { - if err := AddChallengeAchievement(name, uid); err != nil { + if achieved && !ch.Achieved { + if err := AddChallengeAchievement(ch.Name, uid); err != nil { return err } } @@ -57,6 +57,15 @@ func CheckChallengeAndUpdate(name string, uid bson.ObjectId) error { } func CheckChallengeAndUpdateAll(uid bson.ObjectId) error { + challenges, err := model.GetChallengeListWithAchieved(nil, 0, constants.Infinite, "-_id", uid) + if err != nil { + return err + } + for _, ch := range challenges { + if err := CheckChallengeAndUpdate(ch, uid); err != nil { + continue + } + } return nil } diff --git a/backend/services/challenge/create_customized_spider.go b/backend/services/challenge/create_customized_spider.go index cad1aa25..6c61318f 100644 --- a/backend/services/challenge/create_customized_spider.go +++ b/backend/services/challenge/create_customized_spider.go @@ -1,8 +1,23 @@ package challenge +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + type CreateCustomizedSpiderService struct { + UserId bson.ObjectId } func (s *CreateCustomizedSpiderService) Check() (bool, error) { - return true, nil + query := bson.M{ + "user_id": s.UserId, + "type": constants.Customized, + } + _, count, err := model.GetSpiderList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return count > 0, nil } diff --git a/backend/services/challenge/login_7d.go b/backend/services/challenge/login_7d.go index c10e2d38..91540423 100644 --- a/backend/services/challenge/login_7d.go +++ b/backend/services/challenge/login_7d.go @@ -1,8 +1,18 @@ package challenge +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + type Login7dService struct { + UserId bson.ObjectId } func (s *Login7dService) Check() (bool, error) { - return true, nil + days, err := model.GetVisitDays(s.UserId) + if err != nil { + return false, err + } + return days >= 7, nil } diff --git a/backend/services/challenge/run_random.go b/backend/services/challenge/run_random.go index dd9e4a40..ebdc11fe 100644 --- a/backend/services/challenge/run_random.go +++ b/backend/services/challenge/run_random.go @@ -1,8 +1,23 @@ package challenge +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + type RunRandomService struct { + UserId bson.ObjectId } func (s *RunRandomService) Check() (bool, error) { - return true, nil + query := bson.M{ + "user_id": s.UserId, + "run_type": constants.RunTypeRandom, + } + list, err := model.GetTaskList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil } diff --git a/backend/services/spider.go b/backend/services/spider.go index 77bb19ab..0fbc5ebb 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -471,6 +471,7 @@ func InitDemoSpiders () { ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), FileId: bson.ObjectIdHex(constants.ObjectIdNull), Cmd: configData.Cmd, + UserId: bson.ObjectIdHex(constants.ObjectIdNull), } if err := spider.Add(); err != nil { log.Errorf("add spider error: " + err.Error()) @@ -497,6 +498,7 @@ func InitDemoSpiders () { ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), FileId: bson.ObjectIdHex(constants.ObjectIdNull), Config: configData, + UserId: bson.ObjectIdHex(constants.ObjectIdNull), } if err := spider.Add(); err != nil { log.Errorf("add spider error: " + err.Error()) diff --git a/backend/services/user.go b/backend/services/user.go index a01e721b..b01b4923 100644 --- a/backend/services/user.go +++ b/backend/services/user.go @@ -115,3 +115,8 @@ func GetCurrentUser(c *gin.Context) *model.User { data, _ := c.Get("currentUser") return data.(*model.User) } + +func GetCurrentUserId(c *gin.Context) bson.ObjectId { + return GetCurrentUser(c).Id +} + diff --git a/frontend/src/views/challenge/ChallengeList.vue b/frontend/src/views/challenge/ChallengeList.vue index c0bda6f1..3d45bd35 100644 --- a/frontend/src/views/challenge/ChallengeList.vue +++ b/frontend/src/views/challenge/ChallengeList.vue @@ -77,6 +77,7 @@ export default { }, methods: { async getData () { + await this.$request.post('/challenges-check') const res = await this.$request.get('/challenges') this.challenges = res.data.data || [] } From 40ad7fb9f3981932489be6f67cd98ec9cbd5475d Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 19 Mar 2020 18:56:59 +0800 Subject: [PATCH 06/42] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=8C=91=E6=88=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/constants/action.go | 5 +- backend/constants/challenge.go | 19 ++- backend/data/challenge_data.json | 124 +++++++++++++++++- backend/main.go | 1 + backend/model/challenge.go | 2 +- backend/model/task.go | 2 +- backend/model/user.go | 5 +- backend/routes/user.go | 14 +- backend/services/challenge/base.go | 28 ++++ .../challenge/create_configurable_spider.go | 23 ++++ backend/services/challenge/create_nodes.go | 22 ++++ backend/services/challenge/create_schedule.go | 21 +++ backend/services/challenge/create_user.go | 21 +++ backend/services/challenge/install_dep.go | 23 ++++ backend/services/challenge/install_lang.go | 23 ++++ backend/services/challenge/login_180d.go | 18 +++ backend/services/challenge/login_30d.go | 18 +++ backend/services/challenge/login_90d.go | 18 +++ backend/services/challenge/run_random.go | 1 + backend/services/challenge/scrape_100k.go | 24 ++++ backend/services/challenge/scrape_10k.go | 24 ++++ backend/services/challenge/scrape_1k.go | 24 ++++ backend/services/challenge/view_disclaimer.go | 23 ++++ backend/services/user.go | 8 +- .../src/components/Node/NodeInstallation.vue | 6 + .../Node/NodeInstallationMatrix.vue | 8 ++ frontend/src/i18n/zh.js | 1 + frontend/src/store/modules/user.js | 2 +- .../src/views/challenge/ChallengeList.vue | 12 +- frontend/src/views/doc/Disclaimer.vue | 5 + 30 files changed, 504 insertions(+), 21 deletions(-) create mode 100644 backend/services/challenge/create_configurable_spider.go create mode 100644 backend/services/challenge/create_nodes.go create mode 100644 backend/services/challenge/create_schedule.go create mode 100644 backend/services/challenge/create_user.go create mode 100644 backend/services/challenge/install_dep.go create mode 100644 backend/services/challenge/install_lang.go create mode 100644 backend/services/challenge/login_180d.go create mode 100644 backend/services/challenge/login_30d.go create mode 100644 backend/services/challenge/login_90d.go create mode 100644 backend/services/challenge/scrape_100k.go create mode 100644 backend/services/challenge/scrape_10k.go create mode 100644 backend/services/challenge/scrape_1k.go create mode 100644 backend/services/challenge/view_disclaimer.go diff --git a/backend/constants/action.go b/backend/constants/action.go index 31d3655e..389a11bc 100644 --- a/backend/constants/action.go +++ b/backend/constants/action.go @@ -1,5 +1,8 @@ package constants const ( - ActionTypeVisit = "visit" + ActionTypeVisit = "visit" + ActionTypeInstallDep = "install_dep" + ActionTypeInstallLang = "install_lang" + ActionTypeViewDisclaimer = "view_disclaimer" ) diff --git a/backend/constants/challenge.go b/backend/constants/challenge.go index 4cfa9710..5c056e91 100644 --- a/backend/constants/challenge.go +++ b/backend/constants/challenge.go @@ -1,7 +1,20 @@ package constants const ( - ChallengeLogin7d = "login_7d" - ChallengeCreateCustomizedSpider = "create_customized_spider" - ChallengeRunRandom = "run_random" + ChallengeLogin7d = "login_7d" + ChallengeLogin30d = "login_30d" + ChallengeLogin90d = "login_90d" + ChallengeLogin180d = "login_180d" + ChallengeCreateCustomizedSpider = "create_customized_spider" + ChallengeCreateConfigurableSpider = "create_configurable_spider" + ChallengeCreateSchedule = "create_schedule" + ChallengeCreateNodes = "create_nodes" + ChallengeCreateUser = "create_user" + ChallengeRunRandom = "run_random" + ChallengeScrape1k = "scrape_1k" + ChallengeScrape10k = "scrape_10k" + ChallengeScrape100k = "scrape_100k" + ChallengeInstallDep = "install_dep" + ChallengeInstallLang = "install_lang" + ChallengeViewDisclaimer = "view_disclaimer" ) diff --git a/backend/data/challenge_data.json b/backend/data/challenge_data.json index e8cea7da..5a51dc33 100644 --- a/backend/data/challenge_data.json +++ b/backend/data/challenge_data.json @@ -7,22 +7,136 @@ "description_en": "Logged-in for consecutive 7 days to complete the challenge", "difficulty": 1 }, + { + "name": "login_30d", + "title_cn": "连续登录 30 天", + "title_en": "Logged-in for 30 days", + "description_cn": "连续 30 天登录 Crawlab，即可完成挑战！", + "description_en": "Logged-in for consecutive 30 days to complete the challenge", + "difficulty": 2 + }, + { + "name": "login_90d", + "title_cn": "连续登录 90 天", + "title_en": "Logged-in for 90 days", + "description_cn": "连续 90 天登录 Crawlab，即可完成挑战！", + "description_en": "Logged-in for consecutive 90 days to complete the challenge", + "difficulty": 3 + }, + { + "name": "login_180d", + "title_cn": "连续登录 180 天", + "title_en": "Logged-in for 180 days", + "description_cn": "连续 180 天登录 Crawlab，即可完成挑战！", + "description_en": "Logged-in for consecutive 180 days to complete the challenge", + "difficulty": 4 + }, { "name": "create_customized_spider", - "title_cn": "创建一个自定义爬虫", + "title_cn": "创建 1 个自定义爬虫", "title_en": "Create a customized spider", "description_cn": "在爬虫列表中，点击 '添加爬虫'，选择 '自定义爬虫'，输入相应的参数，点击添加，即可完成挑战！", "description_en": "In Spider List page, click 'Add Spider', select 'Customized Spider', enter params, click 'Add' to finish the challenge.", "difficulty": 1, "path": "/spiders" }, + { + "name": "create_configurable_spider", + "title_cn": "创建 1 个可配置爬虫", + "title_en": "Create a configurable spider", + "description_cn": "在爬虫列表中，点击 '添加爬虫'，选择 '可配置爬虫'，输入相应的参数，点击添加，即可完成挑战！", + "description_en": "In Spider List page, click 'Add Spider', select 'Configurable Spider', enter params, click 'Add' to finish the challenge.", + "difficulty": 1, + "path": "/spiders" + }, { "name": "run_random", - "title_cn": "用随机模式运行一个爬虫", - "title_en": "Run a spider in random mode", - "description_cn": "在您创建好的爬虫中，导航到其对应的详情页（爬虫列表中点击爬虫），运行一个爬虫，选择随机模式。", - "description_en": "In your created spiders, navigate to corresponding detail page (click spider in Spider List page), run a spider in random mode.", + "title_cn": "用随机模式成功运行爬虫", + "title_en": "Run a spider in random mode successfully", + "description_cn": "在您创建好的爬虫中，导航到其对应的详情页（爬虫列表中点击爬虫），选择随机模式运行一个爬虫，并能运行成功。", + "description_en": "In your created spiders, navigate to corresponding detail page (click spider in Spider List page), run a spider in random mode successfully.", + "difficulty": 1, + "path": "/spiders" + }, + { + "name": "scrape_1k", + "title_cn": "抓取 1 千条数据", + "title_en": "Scrape 1k records", + "description_cn": "运行您创建好的爬虫，抓取 1 千条及以上的结果数据，即可完成挑战！", + "description_en": "Run your created spiders, scrape 1k and more results to finish the challenge.", "difficulty": 2, "path": "/spiders" + }, + { + "name": "scrape_10k", + "title_cn": "抓取 1 万条数据", + "title_en": "Scrape 10k records", + "description_cn": "运行您创建好的爬虫，抓取 1 万条及以上的结果数据，即可完成挑战！", + "description_en": "Run your created spiders, scrape 10k and more results to finish the challenge.", + "difficulty": 3, + "path": "/spiders" + }, + { + "name": "scrape_100k", + "title_cn": "抓取 10 万条数据", + "title_en": "Scrape 100k records", + "description_cn": "运行您创建好的爬虫，抓取 10 万条及以上的结果数据，即可完成挑战！", + "description_en": "Run your created spiders, scrape 100k and more results to finish the challenge.", + "difficulty": 4, + "path": "/spiders" + }, + { + "name": "create_schedule", + "title_cn": "创建 1 个定时任务", + "title_en": "Create a schedule", + "description_cn": "在定时任务列表中，创建一个定时任务，正确设置好 Cron 表达式，即可完成挑战！", + "description_en": "In Schedule List page, create a schedule and configure cron expression to finish the task.", + "difficulty": 1, + "path": "/schedules" + }, + { + "name": "create_nodes", + "title_cn": "创建 1 个节点集群", + "title_en": "Create a node cluster", + "description_cn": "按照文档的部署指南，部署含有 3 个节点的集群，即可完成挑战！", + "description_en": "Deploy a 3-node cluster according to the deployment guidance in documentation to finish the task.", + "difficulty": 3, + "path": "/nodes" + }, + { + "name": "install_dep", + "title_cn": "安装 1 个依赖", + "title_en": "Install a dependency successfully", + "description_cn": "在 '节点列表->安装' 或 '节点详情->安装' 中，搜索并安装所需的 1 个依赖，即可完成挑战！", + "description_en": "In 'Node List -> Installation' or 'Node Detail -> Installation', search and install a dependency.", + "difficulty": 3, + "path": "/nodes" + }, + { + "name": "install_lang", + "title_cn": "安装 1 个语言环境", + "title_en": "Install a language successfully", + "description_cn": "在 '节点列表->安装' 或 '节点详情->安装' 中，点击安装所需的 1 个语言环境，即可完成挑战！", + "description_en": "In 'Node List -> Installation' or 'Node Detail -> Installation', install a language.", + "difficulty": 3, + "path": "/nodes" + }, + { + "name": "view_disclaimer", + "title_cn": "阅读免责声明", + "title_en": "View disclaimer", + "description_cn": "在左侧菜单栏，点击 '免责声明' 查看其内容，即可完成挑战！", + "description_en": "In the left side menu, click 'Disclaimer' and view its content to finish the challenge.", + "difficulty": 1, + "path": "/disclaimer" + }, + { + "name": "create_user", + "title_cn": "创建 1 个用户", + "title_en": "Create a user", + "description_cn": "在用户管理页面中创建一个新用户，即可完成挑战！", + "description_en": "In User Admin page, create a new user to finish the challenge.", + "difficulty": 1, + "path": "/users" } ] \ No newline at end of file diff --git a/backend/main.go b/backend/main.go index 6550bbbd..ac76a453 100644 --- a/backend/main.go +++ b/backend/main.go @@ -240,6 +240,7 @@ func main() { authGroup.GET("/users/:id", routes.GetUser) // 用户详情 authGroup.POST("/users/:id", routes.PostUser) // 更改用户 authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 + authGroup.PUT("/users-add", routes.PutUser) // 添加用户 authGroup.GET("/me", routes.GetMe) // 获取自己账户 authGroup.POST("/me", routes.PostMe) // 修改自己账户 } diff --git a/backend/model/challenge.go b/backend/model/challenge.go index 083d216b..09f4db89 100644 --- a/backend/model/challenge.go +++ b/backend/model/challenge.go @@ -32,7 +32,7 @@ func (ch *Challenge) Save() error { ch.UpdateTs = time.Now() - if err := c.UpdateId(ch.Id, c); err != nil { + if err := c.UpdateId(ch.Id, ch); err != nil { debug.PrintStack() return err } diff --git a/backend/model/task.go b/backend/model/task.go index 6b2a44c1..1793d0b8 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -26,7 +26,7 @@ type Task struct { TotalDuration float64 `json:"total_duration" bson:"total_duration"` Pid int `json:"pid" bson:"pid"` RunType string `json:"run_type" bson:"run_type"` - ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"` + //ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"` // 前端数据 SpiderName string `json:"spider_name"` diff --git a/backend/model/user.go b/backend/model/user.go index 9dadec0f..074a197a 100644 --- a/backend/model/user.go +++ b/backend/model/user.go @@ -19,8 +19,9 @@ type User struct { Email string `json:"email" bson:"email"` Setting UserSetting `json:"setting" bson:"setting"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } type UserSetting struct { diff --git a/backend/routes/user.go b/backend/routes/user.go index fcca967e..86d46a61 100644 --- a/backend/routes/user.go +++ b/backend/routes/user.go @@ -95,8 +95,11 @@ func PutUser(c *gin.Context) { reqData.Role = constants.RoleNormal } + // UserId + uid := services.GetCurrentUserId(c) + // 添加用户 - if err := services.CreateNewUser(reqData.Username, reqData.Password, reqData.Role, reqData.Email); err != nil { + if err := services.CreateNewUser(reqData.Username, reqData.Password, reqData.Role, reqData.Email, uid); err != nil { HandleError(http.StatusInternalServerError, c, err) return } @@ -120,6 +123,10 @@ func PostUser(c *gin.Context) { return } + if item.UserId.Hex() == "" { + item.UserId = bson.ObjectIdHex(constants.ObjectIdNull) + } + if err := model.UpdateUser(bson.ObjectIdHex(id), item); err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -230,6 +237,11 @@ func PostMe(c *gin.Context) { user.Setting.WechatRobotWebhook = reqBody.Setting.WechatRobotWebhook } user.Setting.EnabledNotifications = reqBody.Setting.EnabledNotifications + + if user.UserId.Hex() == "" { + user.UserId = bson.ObjectIdHex(constants.ObjectIdNull) + } + if err := user.Save(); err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/backend/services/challenge/base.go b/backend/services/challenge/base.go index 9515ff9e..a7758708 100644 --- a/backend/services/challenge/base.go +++ b/backend/services/challenge/base.go @@ -19,10 +19,36 @@ func GetService(name string, uid bson.ObjectId) Service { switch name { case constants.ChallengeLogin7d: return &Login7dService{UserId: uid} + case constants.ChallengeLogin30d: + return &Login30dService{UserId: uid} + case constants.ChallengeLogin90d: + return &Login90dService{UserId: uid} + case constants.ChallengeLogin180d: + return &Login180dService{UserId: uid} case constants.ChallengeCreateCustomizedSpider: return &CreateCustomizedSpiderService{UserId: uid} + case constants.ChallengeCreateConfigurableSpider: + return &CreateConfigurableSpiderService{UserId: uid} + case constants.ChallengeCreateSchedule: + return &CreateScheduleService{UserId: uid} + case constants.ChallengeCreateNodes: + return &CreateNodesService{UserId: uid} case constants.ChallengeRunRandom: return &RunRandomService{UserId: uid} + case constants.ChallengeScrape1k: + return &Scrape1kService{UserId: uid} + case constants.ChallengeScrape10k: + return &Scrape10kService{UserId: uid} + case constants.ChallengeScrape100k: + return &Scrape100kService{UserId: uid} + case constants.ChallengeInstallDep: + return &InstallDepService{UserId: uid} + case constants.ChallengeInstallLang: + return &InstallLangService{UserId: uid} + case constants.ChallengeViewDisclaimer: + return &ViewDisclaimerService{UserId: uid} + case constants.ChallengeCreateUser: + return &CreateUserService{UserId: uid} } return nil } @@ -98,6 +124,8 @@ func InitChallengeService() error { continue } } else { + ch.Id = chDb.Id + ch.CreateTs = chDb.CreateTs if err := ch.Save(); err != nil { log.Errorf(err.Error()) debug.PrintStack() diff --git a/backend/services/challenge/create_configurable_spider.go b/backend/services/challenge/create_configurable_spider.go new file mode 100644 index 00000000..45e969f7 --- /dev/null +++ b/backend/services/challenge/create_configurable_spider.go @@ -0,0 +1,23 @@ +package challenge + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type CreateConfigurableSpiderService struct { + UserId bson.ObjectId +} + +func (s *CreateConfigurableSpiderService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + "type": constants.Configurable, + } + _, count, err := model.GetSpiderList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return count > 0, nil +} diff --git a/backend/services/challenge/create_nodes.go b/backend/services/challenge/create_nodes.go new file mode 100644 index 00000000..42ec25f7 --- /dev/null +++ b/backend/services/challenge/create_nodes.go @@ -0,0 +1,22 @@ +package challenge + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type CreateNodesService struct { + UserId bson.ObjectId +} + +func (s *CreateNodesService) Check() (bool, error) { + query := bson.M{ + "status": constants.StatusOnline, + } + list, err := model.GetScheduleList(query) + if err != nil { + return false, err + } + return len(list) >= 3, nil +} diff --git a/backend/services/challenge/create_schedule.go b/backend/services/challenge/create_schedule.go new file mode 100644 index 00000000..3e0ce0e1 --- /dev/null +++ b/backend/services/challenge/create_schedule.go @@ -0,0 +1,21 @@ +package challenge + +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type CreateScheduleService struct { + UserId bson.ObjectId +} + +func (s *CreateScheduleService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + } + list, err := model.GetScheduleList(query) + if err != nil { + return false, err + } + return len(list) > 0, nil +} diff --git a/backend/services/challenge/create_user.go b/backend/services/challenge/create_user.go new file mode 100644 index 00000000..e0272801 --- /dev/null +++ b/backend/services/challenge/create_user.go @@ -0,0 +1,21 @@ +package challenge + +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type CreateUserService struct { + UserId bson.ObjectId +} + +func (s *CreateUserService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + } + list, err := model.GetUserList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil +} diff --git a/backend/services/challenge/install_dep.go b/backend/services/challenge/install_dep.go new file mode 100644 index 00000000..4730249e --- /dev/null +++ b/backend/services/challenge/install_dep.go @@ -0,0 +1,23 @@ +package challenge + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type InstallDepService struct { + UserId bson.ObjectId +} + +func (s *InstallDepService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + "type": constants.ActionTypeInstallDep, + } + list, err := model.GetActionList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil +} diff --git a/backend/services/challenge/install_lang.go b/backend/services/challenge/install_lang.go new file mode 100644 index 00000000..15732a2f --- /dev/null +++ b/backend/services/challenge/install_lang.go @@ -0,0 +1,23 @@ +package challenge + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type InstallLangService struct { + UserId bson.ObjectId +} + +func (s *InstallLangService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + "type": constants.ActionTypeInstallLang, + } + list, err := model.GetActionList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil +} diff --git a/backend/services/challenge/login_180d.go b/backend/services/challenge/login_180d.go new file mode 100644 index 00000000..96cc9e26 --- /dev/null +++ b/backend/services/challenge/login_180d.go @@ -0,0 +1,18 @@ +package challenge + +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type Login180dService struct { + UserId bson.ObjectId +} + +func (s *Login180dService) Check() (bool, error) { + days, err := model.GetVisitDays(s.UserId) + if err != nil { + return false, err + } + return days >= 180, nil +} diff --git a/backend/services/challenge/login_30d.go b/backend/services/challenge/login_30d.go new file mode 100644 index 00000000..5234d5fe --- /dev/null +++ b/backend/services/challenge/login_30d.go @@ -0,0 +1,18 @@ +package challenge + +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type Login30dService struct { + UserId bson.ObjectId +} + +func (s *Login30dService) Check() (bool, error) { + days, err := model.GetVisitDays(s.UserId) + if err != nil { + return false, err + } + return days >= 30, nil +} diff --git a/backend/services/challenge/login_90d.go b/backend/services/challenge/login_90d.go new file mode 100644 index 00000000..a8526b87 --- /dev/null +++ b/backend/services/challenge/login_90d.go @@ -0,0 +1,18 @@ +package challenge + +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type Login90dService struct { + UserId bson.ObjectId +} + +func (s *Login90dService) Check() (bool, error) { + days, err := model.GetVisitDays(s.UserId) + if err != nil { + return false, err + } + return days >= 90, nil +} diff --git a/backend/services/challenge/run_random.go b/backend/services/challenge/run_random.go index ebdc11fe..aad4fe21 100644 --- a/backend/services/challenge/run_random.go +++ b/backend/services/challenge/run_random.go @@ -14,6 +14,7 @@ func (s *RunRandomService) Check() (bool, error) { query := bson.M{ "user_id": s.UserId, "run_type": constants.RunTypeRandom, + "status": constants.StatusFinished, } list, err := model.GetTaskList(query, 0, 1, "-_id") if err != nil { diff --git a/backend/services/challenge/scrape_100k.go b/backend/services/challenge/scrape_100k.go new file mode 100644 index 00000000..68a90eda --- /dev/null +++ b/backend/services/challenge/scrape_100k.go @@ -0,0 +1,24 @@ +package challenge + +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type Scrape100kService struct { + UserId bson.ObjectId +} + +func (s *Scrape100kService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + "result_count": bson.M{ + "$gte": 100000, + }, + } + list, err := model.GetTaskList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil +} diff --git a/backend/services/challenge/scrape_10k.go b/backend/services/challenge/scrape_10k.go new file mode 100644 index 00000000..ae70b450 --- /dev/null +++ b/backend/services/challenge/scrape_10k.go @@ -0,0 +1,24 @@ +package challenge + +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type Scrape10kService struct { + UserId bson.ObjectId +} + +func (s *Scrape10kService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + "result_count": bson.M{ + "$gte": 10000, + }, + } + list, err := model.GetTaskList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil +} diff --git a/backend/services/challenge/scrape_1k.go b/backend/services/challenge/scrape_1k.go new file mode 100644 index 00000000..cad2469f --- /dev/null +++ b/backend/services/challenge/scrape_1k.go @@ -0,0 +1,24 @@ +package challenge + +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type Scrape1kService struct { + UserId bson.ObjectId +} + +func (s *Scrape1kService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + "result_count": bson.M{ + "$gte": 1000, + }, + } + list, err := model.GetTaskList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil +} diff --git a/backend/services/challenge/view_disclaimer.go b/backend/services/challenge/view_disclaimer.go new file mode 100644 index 00000000..fc9fe21c --- /dev/null +++ b/backend/services/challenge/view_disclaimer.go @@ -0,0 +1,23 @@ +package challenge + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + +type ViewDisclaimerService struct { + UserId bson.ObjectId +} + +func (s *ViewDisclaimerService) Check() (bool, error) { + query := bson.M{ + "user_id": s.UserId, + "type": constants.ActionTypeViewDisclaimer, + } + list, err := model.GetActionList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil +} diff --git a/backend/services/user.go b/backend/services/user.go index b01b4923..5e283044 100644 --- a/backend/services/user.go +++ b/backend/services/user.go @@ -14,7 +14,7 @@ import ( ) func InitUserService() error { - _ = CreateNewUser("admin", "admin", constants.RoleAdmin, "") + _ = CreateNewUser("admin", "admin", constants.RoleAdmin, "", bson.ObjectIdHex(constants.ObjectIdNull)) return nil } @@ -90,12 +90,13 @@ func CheckToken(tokenStr string) (user model.User, err error) { return } -func CreateNewUser(username string, password string, role string, email string) error { +func CreateNewUser(username string, password string, role string, email string, uid bson.ObjectId) error { user := model.User{ Username: strings.ToLower(username), Password: utils.EncryptPassword(password), Role: role, Email: email, + UserId: uid, Setting: model.UserSetting{ NotificationTrigger: constants.NotificationTriggerNever, EnabledNotifications: []string{ @@ -112,11 +113,10 @@ func CreateNewUser(username string, password string, role string, email string) } func GetCurrentUser(c *gin.Context) *model.User { - data, _ := c.Get("currentUser") + data, _ := c.Get(constants.ContextUser) return data.(*model.User) } func GetCurrentUserId(c *gin.Context) bson.ObjectId { return GetCurrentUser(c).Id } - diff --git a/frontend/src/components/Node/NodeInstallation.vue b/frontend/src/components/Node/NodeInstallation.vue index d5813baa..d7a29186 100644 --- a/frontend/src/components/Node/NodeInstallation.vue +++ b/frontend/src/components/Node/NodeInstallation.vue @@ -262,6 +262,9 @@ export default { }) dep.installed = true } + this.$request.put('/actions', { + type: 'install_dep' + }) this.$set(this.depLoadingDict, name, false) this.$st.sendEv('节点详情', '安装', '安装依赖') }, @@ -312,6 +315,9 @@ export default { message: this.$t('You have successfully installed a language: ') + this.activeLang.name }) } + this.$request.put('/actions', { + type: 'install_lang' + }) this.isLoadingInstallLang = false this.$st.sendEv('节点详情', '安装', '安装语言') }, diff --git a/frontend/src/components/Node/NodeInstallationMatrix.vue b/frontend/src/components/Node/NodeInstallationMatrix.vue index 8c628914..adcff113 100644 --- a/frontend/src/components/Node/NodeInstallationMatrix.vue +++ b/frontend/src/components/Node/NodeInstallationMatrix.vue @@ -266,6 +266,7 @@ export default { await Promise.all(this.nodeList.map(async n => { if (n.status !== 'online') return const res = await this.$request.get(`/nodes/${n._id}/langs`) + if (!res.data.data) return res.data.data.forEach(l => { const key = n._id + '|' + l.executable_name this.$set(this.langsDataDict, key, l) @@ -280,6 +281,7 @@ export default { await Promise.all(this.nodeList.map(async n => { if (n.status !== 'online') return const res = await this.$request.get(`/nodes/${n._id}/deps/installed`, { lang: this.activeLang }) + if (!res.data.data) return res.data.data.forEach(d => { depsSet.add(d.name) const key = n._id + '|' + d.name @@ -319,6 +321,9 @@ export default { setTimeout(() => { this.getLangsData() }, 1000) + this.$request.put('/actions', { + type: 'install_lang' + }) this.$st.sendEv('节点列表', '安装', '安装语言') }, async onInstallLangAll (langLabel, ev) { @@ -372,6 +377,9 @@ export default { }) this.$set(this.depsDataDict, key, 'installed') } + this.$request.put('/actions', { + type: 'install_dep' + }) this.$st.sendEv('节点列表', '安装', '安装依赖') }, async uninstallDep (node, dep) { diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index e677d90b..2695c850 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -592,6 +592,7 @@ docker run -d --restart always --name crawlab_worker \\ 'Are you sure to stop selected items?': '您是否确认停止所选项？', 'Sent signals to cancel selected tasks': '已经向所选任务发送取消任务信号', 'Copied successfully': '已成功复制', + 'You have started the challenge.': '您已开始挑战', // 其他 'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧' diff --git a/frontend/src/store/modules/user.js b/frontend/src/store/modules/user.js index 4bb6e918..0e812134 100644 --- a/frontend/src/store/modules/user.js +++ b/frontend/src/store/modules/user.js @@ -152,7 +152,7 @@ const user = { // 添加用户 addUser ({ dispatch, commit, state }) { - return request.put('/users', state.userForm) + return request.put('/users-add', state.userForm) }, // 新增全局变量 addGlobalVariable ({ commit, state }) { diff --git a/frontend/src/views/challenge/ChallengeList.vue b/frontend/src/views/challenge/ChallengeList.vue index 3d45bd35..551c1354 100644 --- a/frontend/src/views/challenge/ChallengeList.vue +++ b/frontend/src/views/challenge/ChallengeList.vue @@ -49,6 +49,7 @@ size="mini" type="primary" icon="el-icon-s-flag" + @click="onStartChallenge(c)" > {{$t('Start Challenge')}} @@ -80,6 +81,13 @@ export default { await this.$request.post('/challenges-check') const res = await this.$request.get('/challenges') this.challenges = res.data.data || [] + }, + onStartChallenge (c) { + if (c.path) { + this.$router.push(c.path) + } else { + this.$message.success(this.$t('You have started the challenge.')) + } } }, async created () { @@ -96,8 +104,8 @@ export default { } .challenge-list .challenge-item { - flex-basis: 240px; - width: 240px; + flex-basis: 280px; + width: 280px; margin: 10px; } diff --git a/frontend/src/views/doc/Disclaimer.vue b/frontend/src/views/doc/Disclaimer.vue index b05f8e96..d66c6614 100644 --- a/frontend/src/views/doc/Disclaimer.vue +++ b/frontend/src/views/doc/Disclaimer.vue @@ -60,6 +60,11 @@ This Disclaimer and privacy protection statement (hereinafter referred to as "di 8. 传播:任何公司或个人在网络上发布，传播我们软件的行为都是允许的，但因公司或个人传播软件可能造成的任何法律和刑事事件 Crawlab 开发组不负任何责任。 ` } + }, + mounted () { + this.$request.put('/actions', { + type: 'view_disclaimer' + }) } } From 6b2d51fb1ef16da6fb9cdd5094eed87ccbbdea04 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Mar 2020 08:27:54 +0800 Subject: [PATCH 07/42] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=AF=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E6=97=A0=E6=B3=95=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/routes/config_spider.go | 17 +++++++++++++++++ backend/routes/spider.go | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go index ac6a11e0..4c65964e 100644 --- a/backend/routes/config_spider.go +++ b/backend/routes/config_spider.go @@ -51,6 +51,9 @@ func PutConfigSpider(c *gin.Context) { // 将FileId置空 spider.FileId = bson.ObjectIdHex(constants.ObjectIdNull) + // UserId + spider.UserId = services.GetCurrentUserId(c) + // 创建爬虫目录 spiderDir := filepath.Join(viper.GetString("spider.path"), spider.Name) if utils.Exists(spiderDir) { @@ -109,8 +112,12 @@ func UploadConfigSpider(c *gin.Context) { spider, err := model.GetSpider(bson.ObjectIdHex(id)) if err != nil { HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("cannot find spider (id: %s)", id)) + return } + // UserId + spider.UserId = services.GetCurrentUserId(c) + // 获取上传文件 file, header, err := c.Request.FormFile("file") if err != nil { @@ -205,6 +212,11 @@ func PostConfigSpiderSpiderfile(c *gin.Context) { return } + // UserId + if !spider.UserId.Valid() { + spider.UserId = bson.ObjectIdHex(constants.ObjectIdNull) + } + // 反序列化 var configData entity.ConfigSpiderData if err := yaml.Unmarshal([]byte(content), &configData); err != nil { @@ -247,6 +259,11 @@ func PostConfigSpiderConfig(c *gin.Context) { return } + // UserId + if !spider.UserId.Valid() { + spider.UserId = bson.ObjectIdHex(constants.ObjectIdNull) + } + // 反序列化配置数据 var configData entity.ConfigSpiderData if err := c.ShouldBindJSON(&configData); err != nil { diff --git a/backend/routes/spider.go b/backend/routes/spider.go index b136070b..8f7a513b 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -126,6 +126,11 @@ func PostSpider(c *gin.Context) { return } + // UserId + if !item.UserId.Valid() { + item.UserId = bson.ObjectIdHex(constants.ObjectIdNull) + } + if err := model.UpdateSpider(bson.ObjectIdHex(id), item); err != nil { HandleError(http.StatusInternalServerError, c, err) return From 1e2b580ed49c9ac2e0a98e5a4d783b1677fd7771 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Mar 2020 08:33:30 +0800 Subject: [PATCH 08/42] =?UTF-8?q?=E5=8A=A0=E5=85=A5ScheduleId=E5=88=B0Task?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/task.go | 2 +- backend/routes/spider.go | 31 +++++++++++---------- backend/routes/task.go | 31 +++++++++++---------- backend/services/challenge/run_random.go | 7 ++--- backend/services/schedule.go | 34 ++++++++++++++---------- 5 files changed, 59 insertions(+), 46 deletions(-) diff --git a/backend/model/task.go b/backend/model/task.go index 1793d0b8..6b2a44c1 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -26,7 +26,7 @@ type Task struct { TotalDuration float64 `json:"total_duration" bson:"total_duration"` Pid int `json:"pid" bson:"pid"` RunType string `json:"run_type" bson:"run_type"` - //ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"` + ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"` // 前端数据 SpiderName string `json:"spider_name"` diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 8f7a513b..049e69b2 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -627,11 +627,12 @@ func RunSelectedSpider(c *gin.Context) { } for _, node := range nodes { t := model.Task{ - SpiderId: taskParam.SpiderId, - NodeId: node.Id, - Param: taskParam.Param, - UserId: services.GetCurrentUserId(c), - RunType: constants.RunTypeAllNodes, + SpiderId: taskParam.SpiderId, + NodeId: node.Id, + Param: taskParam.Param, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeAllNodes, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), } id, err := services.AddTask(t) @@ -645,10 +646,11 @@ func RunSelectedSpider(c *gin.Context) { } else if reqBody.RunType == constants.RunTypeRandom { // 随机 t := model.Task{ - SpiderId: taskParam.SpiderId, - Param: taskParam.Param, - UserId: services.GetCurrentUserId(c), - RunType: constants.RunTypeRandom, + SpiderId: taskParam.SpiderId, + Param: taskParam.Param, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeRandom, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), } id, err := services.AddTask(t) if err != nil { @@ -660,11 +662,12 @@ func RunSelectedSpider(c *gin.Context) { // 指定节点 for _, nodeId := range reqBody.NodeIds { t := model.Task{ - SpiderId: taskParam.SpiderId, - NodeId: nodeId, - Param: taskParam.Param, - UserId: services.GetCurrentUserId(c), - RunType: constants.RunTypeSelectedNodes, + SpiderId: taskParam.SpiderId, + NodeId: nodeId, + Param: taskParam.Param, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeSelectedNodes, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), } id, err := services.AddTask(t) diff --git a/backend/routes/task.go b/backend/routes/task.go index ecfbc71f..7e772ca5 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -112,11 +112,12 @@ func PutTask(c *gin.Context) { } for _, node := range nodes { t := model.Task{ - SpiderId: reqBody.SpiderId, - NodeId: node.Id, - Param: reqBody.Param, - UserId: services.GetCurrentUserId(c), - RunType: constants.RunTypeAllNodes, + SpiderId: reqBody.SpiderId, + NodeId: node.Id, + Param: reqBody.Param, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeAllNodes, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), } id, err := services.AddTask(t) @@ -130,10 +131,11 @@ func PutTask(c *gin.Context) { } else if reqBody.RunType == constants.RunTypeRandom { // 随机 t := model.Task{ - SpiderId: reqBody.SpiderId, - Param: reqBody.Param, - UserId: services.GetCurrentUserId(c), - RunType: constants.RunTypeRandom, + SpiderId: reqBody.SpiderId, + Param: reqBody.Param, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeRandom, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), } id, err := services.AddTask(t) if err != nil { @@ -145,11 +147,12 @@ func PutTask(c *gin.Context) { // 指定节点 for _, nodeId := range reqBody.NodeIds { t := model.Task{ - SpiderId: reqBody.SpiderId, - NodeId: nodeId, - Param: reqBody.Param, - UserId: services.GetCurrentUserId(c), - RunType: constants.RunTypeSelectedNodes, + SpiderId: reqBody.SpiderId, + NodeId: nodeId, + Param: reqBody.Param, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeSelectedNodes, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), } id, err := services.AddTask(t) diff --git a/backend/services/challenge/run_random.go b/backend/services/challenge/run_random.go index aad4fe21..30c63f0c 100644 --- a/backend/services/challenge/run_random.go +++ b/backend/services/challenge/run_random.go @@ -12,9 +12,10 @@ type RunRandomService struct { func (s *RunRandomService) Check() (bool, error) { query := bson.M{ - "user_id": s.UserId, - "run_type": constants.RunTypeRandom, - "status": constants.StatusFinished, + "user_id": s.UserId, + "run_type": constants.RunTypeRandom, + "status": constants.StatusFinished, + "schedule_id": bson.ObjectIdHex(constants.ObjectIdNull), } list, err := model.GetTaskList(query, 0, 1, "-_id") if err != nil { diff --git a/backend/services/schedule.go b/backend/services/schedule.go index ad41b969..78d430b1 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -51,11 +51,13 @@ func AddScheduleTask(s model.Schedule) func() { } for _, node := range nodes { t := model.Task{ - Id: id.String(), - SpiderId: s.SpiderId, - NodeId: node.Id, - Param: param, - UserId: s.UserId, + Id: id.String(), + SpiderId: s.SpiderId, + NodeId: node.Id, + Param: param, + UserId: s.UserId, + RunType: constants.RunTypeAllNodes, + ScheduleId: s.Id, } if _, err := AddTask(t); err != nil { @@ -65,10 +67,12 @@ func AddScheduleTask(s model.Schedule) func() { } else if s.RunType == constants.RunTypeRandom { // 随机 t := model.Task{ - Id: id.String(), - SpiderId: s.SpiderId, - Param: param, - UserId: s.UserId, + Id: id.String(), + SpiderId: s.SpiderId, + Param: param, + UserId: s.UserId, + RunType: constants.RunTypeRandom, + ScheduleId: s.Id, } if _, err := AddTask(t); err != nil { log.Errorf(err.Error()) @@ -79,11 +83,13 @@ func AddScheduleTask(s model.Schedule) func() { // 指定节点 for _, nodeId := range s.NodeIds { t := model.Task{ - Id: id.String(), - SpiderId: s.SpiderId, - NodeId: nodeId, - Param: param, - UserId: s.UserId, + Id: id.String(), + SpiderId: s.SpiderId, + NodeId: nodeId, + Param: param, + UserId: s.UserId, + RunType: constants.RunTypeSelectedNodes, + ScheduleId: s.Id, } if _, err := AddTask(t); err != nil { From fa62e1a2dcbbd395a841d3f12b1b6ecf7ded5653 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 20 Mar 2020 17:43:11 +0800 Subject: [PATCH 09/42] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=9D=83=E9=99=90?= =?UTF-8?q?=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/constants/auth.go | 7 + backend/main.go | 8 ++ backend/model/project.go | 17 ++- backend/model/schedule.go | 22 +++- backend/model/spider.go | 43 +++++- backend/model/task.go | 10 ++ backend/routes/{projects.go => project.go} | 9 +- backend/routes/schedule.go | 7 +- backend/routes/spider.go | 32 +++-- backend/routes/stats.go | 9 +- backend/routes/task.go | 5 +- backend/services/auth.go | 20 +++ backend/services/clean.go | 122 ++++++++++++++++++ backend/services/spider.go | 3 + backend/services/user.go | 8 ++ frontend/src/components/File/FileDetail.vue | 28 +++- frontend/src/components/File/FileList.vue | 20 ++- .../components/InfoView/SpiderInfoView.vue | 44 +++++-- frontend/src/i18n/zh.js | 5 + .../src/views/layout/components/Navbar.vue | 2 + frontend/src/views/login/index.vue | 4 + frontend/src/views/project/ProjectList.vue | 11 +- frontend/src/views/schedule/ScheduleList.vue | 5 +- frontend/src/views/spider/SpiderDetail.vue | 2 +- frontend/src/views/spider/SpiderList.vue | 61 +++++++-- frontend/src/views/task/TaskList.vue | 3 +- 26 files changed, 437 insertions(+), 70 deletions(-) create mode 100644 backend/constants/auth.go rename backend/routes/{projects.go => project.go} (95%) create mode 100644 backend/services/auth.go create mode 100644 backend/services/clean.go diff --git a/backend/constants/auth.go b/backend/constants/auth.go new file mode 100644 index 00000000..136391a0 --- /dev/null +++ b/backend/constants/auth.go @@ -0,0 +1,7 @@ +package constants + +const ( + OwnerTypeAll = "all" + OwnerTypeMe = "me" + OwnerTypePublic = "public" +) diff --git a/backend/main.go b/backend/main.go index ac76a453..110bdecc 100644 --- a/backend/main.go +++ b/backend/main.go @@ -100,6 +100,14 @@ func main() { panic(err) } log.Info("initialized challenge service successfully") + + // 初始化清理服务 + if err := services.InitCleanService(); err != nil { + log.Error("init clean service error:" + err.Error()) + debug.PrintStack() + panic(err) + } + log.Info("initialized clean service successfully") } // 初始化任务执行器 diff --git a/backend/model/project.go b/backend/model/project.go index bc7ca35e..09f52f41 100644 --- a/backend/model/project.go +++ b/backend/model/project.go @@ -16,11 +16,12 @@ type Project struct { Tags []string `json:"tags" bson:"tags"` // 前端展示 - Spiders []Spider `json:"spiders" bson:"spiders"` + Spiders []Spider `json:"spiders" bson:"spiders"` + Username string `json:"username" bson:"username"` UserId bson.ObjectId `json:"user_id" bson:"user_id"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } func (p *Project) Save() error { @@ -90,15 +91,21 @@ func GetProject(id bson.ObjectId) (Project, error) { return p, nil } -func GetProjectList(filter interface{}, skip int, sortKey string) ([]Project, error) { +func GetProjectList(filter interface{}, sortKey string) ([]Project, error) { s, c := database.GetCol("projects") defer s.Close() var projects []Project - if err := c.Find(filter).Skip(skip).Limit(constants.Infinite).Sort(sortKey).All(&projects); err != nil { + if err := c.Find(filter).Sort(sortKey).All(&projects); err != nil { debug.PrintStack() return projects, err } + + for i, p := range projects { + // 获取用户名称 + user, _ := GetUser(p.UserId) + projects[i].Username = user.Username + } return projects, nil } diff --git a/backend/model/schedule.go b/backend/model/schedule.go index a23b6973..ee4028af 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -29,6 +29,7 @@ type Schedule struct { // 前端展示 SpiderName string `json:"spider_name" bson:"spider_name"` + Username string `json:"user_name" bson:"user_name"` Nodes []Node `json:"nodes" bson:"nodes"` Message string `json:"message" bson:"message"` @@ -83,6 +84,10 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) { schedule.SpiderName = spider.Name } + // 获取用户名称 + user, _ := GetUser(schedule.UserId) + schedule.Username = user.Username + schs = append(schs, schedule) } return schs, nil @@ -92,11 +97,16 @@ func GetSchedule(id bson.ObjectId) (Schedule, error) { s, c := database.GetCol("schedules") defer s.Close() - var result Schedule - if err := c.FindId(id).One(&result); err != nil { - return result, err + var schedule Schedule + if err := c.FindId(id).One(&schedule); err != nil { + return schedule, err } - return result, nil + + // 获取用户名称 + user, _ := GetUser(schedule.UserId) + schedule.Username = user.Username + + return schedule, nil } func UpdateSchedule(id bson.ObjectId, item Schedule) error { @@ -147,11 +157,11 @@ func RemoveSchedule(id bson.ObjectId) error { return nil } -func GetScheduleCount() (int, error) { +func GetScheduleCount(filter interface{}) (int, error) { s, c := database.GetCol("schedules") defer s.Close() - count, err := c.Count() + count, err := c.Find(filter).Count() if err != nil { return 0, err } diff --git a/backend/model/spider.go b/backend/model/spider.go index 5bc2579b..9a709b41 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -33,6 +33,7 @@ type Spider struct { Remark string `json:"remark" bson:"remark"` // 备注 Src string `json:"src" bson:"src"` // 源码位置 ProjectId bson.ObjectId `json:"project_id" bson:"project_id"` // 项目ID + IsPublic bool `json:"is_public" bson:"is_public"` // 是否公开 // 自定义爬虫 Cmd string `json:"cmd" bson:"cmd"` // 执行命令 @@ -63,6 +64,7 @@ type Spider struct { LastStatus string `json:"last_status"` // 最后执行状态 Config entity.ConfigSpiderData `json:"config"` // 可配置爬虫配置 LatestTasks []Task `json:"latest_tasks"` // 最近任务列表 + Username string `json:"username""` // 时间 UserId bson.ObjectId `json:"user_id" bson:"user_id"` @@ -83,6 +85,7 @@ func (spider *Spider) Save() error { } if err := c.UpdateId(spider.Id, spider); err != nil { + log.Errorf(err.Error()) debug.PrintStack() return err } @@ -182,10 +185,22 @@ func GetSpiderList(filter interface{}, skip int, limit int, sortStr string) ([]S continue } + // 获取用户 + var user User + if spider.UserId.Valid() { + user, err = GetUser(spider.UserId) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } + // 赋值 spiders[i].LastRunTs = task.CreateTs spiders[i].LastStatus = task.Status spiders[i].LatestTasks = latestTasks + spiders[i].Username = user.Username } count, _ := c.Find(filter).Count() @@ -221,13 +236,21 @@ func GetSpiderByName(name string) Spider { s, c := database.GetCol("spiders") defer s.Close() - var result Spider - if err := c.Find(bson.M{"name": name}).One(&result); err != nil && err != mgo.ErrNotFound { + var spider Spider + if err := c.Find(bson.M{"name": name}).One(&spider); err != nil && err != mgo.ErrNotFound { log.Errorf("get spider error: %s, spider_name: %s", err.Error(), name) //debug.PrintStack() - return result + return spider } - return result + + // 获取用户 + var user User + if spider.UserId.Valid() { + user, _ = GetUser(spider.UserId) + } + spider.Username = user.Username + + return spider } // 获取爬虫(根据ID) @@ -253,6 +276,14 @@ func GetSpider(id bson.ObjectId) (Spider, error) { } spider.Config = config } + + // 获取用户名称 + var user User + if spider.UserId.Valid() { + user, _ = GetUser(spider.UserId) + } + spider.Username = user.Username + return spider, nil } @@ -324,11 +355,11 @@ func RemoveAllSpider() error { } // 获取爬虫总数 -func GetSpiderCount() (int, error) { +func GetSpiderCount(filter interface{}) (int, error) { s, c := database.GetCol("spiders") defer s.Close() - count, err := c.Count() + count, err := c.Find(filter).Count() if err != nil { return 0, err } diff --git a/backend/model/task.go b/backend/model/task.go index 6b2a44c1..75edd631 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -31,6 +31,7 @@ type Task struct { // 前端数据 SpiderName string `json:"spider_name"` NodeName string `json:"node_name"` + Username string `json:"username"` UserId bson.ObjectId `json:"user_id" bson:"user_id"` CreateTs time.Time `json:"create_ts" bson:"create_ts"` @@ -128,6 +129,10 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas if node, err := task.GetNode(); err == nil { tasks[i].NodeName = node.Name } + + // 获取用户名称 + user, _ := GetUser(task.UserId) + task.Username = user.Username } return tasks, nil } @@ -156,6 +161,11 @@ func GetTask(id string) (Task, error) { debug.PrintStack() return task, err } + + // 获取用户名称 + user, _ := GetUser(task.UserId) + task.Username = user.Username + return task, nil } diff --git a/backend/routes/projects.go b/backend/routes/project.go similarity index 95% rename from backend/routes/projects.go rename to backend/routes/project.go index 34b2d7f4..f0dd1198 100644 --- a/backend/routes/projects.go +++ b/backend/routes/project.go @@ -4,6 +4,7 @@ import ( "crawlab/constants" "crawlab/database" "crawlab/model" + "crawlab/services" "github.com/gin-gonic/gin" "github.com/globalsign/mgo/bson" "net/http" @@ -18,8 +19,11 @@ func GetProjectList(c *gin.Context) { query["tags"] = tag } + // 获取校验 + query = services.GetAuthQuery(query, c) + // 获取列表 - projects, err := model.GetProjectList(query, 0, "+_id") + projects, err := model.GetProjectList(query, "+_id") if err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -74,6 +78,9 @@ func PutProject(c *gin.Context) { return } + // UserId + p.UserId = services.GetCurrentUserId(c) + if err := p.Add(); err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index 601d467f..27ad7825 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -9,7 +9,12 @@ import ( ) func GetScheduleList(c *gin.Context) { - results, err := model.GetScheduleList(nil) + query := bson.M{} + + // 获取校验 + query = services.GetAuthQuery(query, c) + + results, err := model.GetScheduleList(query) if err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 049e69b2..6f4d88b3 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -29,13 +29,14 @@ import ( // ======== 爬虫管理 ======== func GetSpiderList(c *gin.Context) { - pageNum, _ := c.GetQuery("page_num") - pageSize, _ := c.GetQuery("page_size") - keyword, _ := c.GetQuery("keyword") - pid, _ := c.GetQuery("project_id") - t, _ := c.GetQuery("type") - sortKey, _ := c.GetQuery("sort_key") - sortDirection, _ := c.GetQuery("sort_direction") + pageNum := c.Query("page_num") + pageSize := c.Query("page_size") + keyword := c.Query("keyword") + pid := c.Query("project_id") + t := c.Query("type") + sortKey := c.Query("sort_key") + sortDirection := c.Query("sort_direction") + ownerType := c.Query("owner_type") // 筛选-名称 filter := bson.M{ @@ -65,6 +66,21 @@ func GetSpiderList(c *gin.Context) { filter["project_id"] = bson.ObjectIdHex(pid) } + // 筛选-用户 + if ownerType == constants.OwnerTypeAll { + user := services.GetCurrentUser(c) + if user.Role == constants.RoleNormal { + filter["$or"] = []bson.M{ + {"user_id": services.GetCurrentUserId(c)}, + {"is_public": true}, + } + } + } else if ownerType == constants.OwnerTypeMe { + filter["user_id"] = services.GetCurrentUserId(c) + } else if ownerType == constants.OwnerTypePublic { + filter["is_public"] = true + } + // 排序 sortStr := "-_id" if sortKey != "" && sortDirection != "" { @@ -815,7 +831,7 @@ func GetSpiderStats(c *gin.Context) { overview.AvgWaitDuration = overview.TotalWaitDuration / taskCount overview.AvgRuntimeDuration = overview.TotalRuntimeDuration / taskCount - items, err := model.GetDailyTaskStats(bson.M{"spider_id": spider.Id}) + items, err := model.GetDailyTaskStats(bson.M{"spider_id": spider.Id, "user_id": bson.M{"user_id": services.GetCurrentUserId(c)}}) if err != nil { log.Errorf(err.Error()) HandleError(http.StatusInternalServerError, c, err) diff --git a/backend/routes/stats.go b/backend/routes/stats.go index 8590bbd7..497083e5 100644 --- a/backend/routes/stats.go +++ b/backend/routes/stats.go @@ -3,6 +3,7 @@ package routes import ( "crawlab/constants" "crawlab/model" + "crawlab/services" "github.com/gin-gonic/gin" "github.com/globalsign/mgo/bson" "net/http" @@ -22,7 +23,7 @@ func GetHomeStats(c *gin.Context) { } // 任务总数 - taskCount, err := model.GetTaskCount(nil) + taskCount, err := model.GetTaskCount(bson.M{"user_id": services.GetCurrentUserId(c)}) if err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -36,21 +37,21 @@ func GetHomeStats(c *gin.Context) { } // 爬虫总数 - spiderCount, err := model.GetSpiderCount() + spiderCount, err := model.GetSpiderCount(bson.M{"user_id": services.GetCurrentUserId(c)}) if err != nil { HandleError(http.StatusInternalServerError, c, err) return } // 定时任务数 - scheduleCount, err := model.GetScheduleCount() + scheduleCount, err := model.GetScheduleCount(bson.M{"user_id": services.GetCurrentUserId(c)}) if err != nil { HandleError(http.StatusInternalServerError, c, err) return } // 每日任务数 - items, err := model.GetDailyTaskStats(bson.M{}) + items, err := model.GetDailyTaskStats(bson.M{"user_id": services.GetCurrentUserId(c)}) if err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/backend/routes/task.go b/backend/routes/task.go index 7e772ca5..2ab1a046 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -47,11 +47,14 @@ func GetTaskList(c *gin.Context) { if data.SpiderId != "" { query["spider_id"] = bson.ObjectIdHex(data.SpiderId) } - //新增根据任务状态获取task列表 + // 根据任务状态获取task列表 if data.Status != "" { query["status"] = data.Status } + // 获取校验 + query = services.GetAuthQuery(query, c) + // 获取任务列表 tasks, err := model.GetTaskList(query, (data.PageNum-1)*data.PageSize, data.PageSize, "-create_ts") if err != nil { diff --git a/backend/services/auth.go b/backend/services/auth.go new file mode 100644 index 00000000..096d9f14 --- /dev/null +++ b/backend/services/auth.go @@ -0,0 +1,20 @@ +package services + +import ( + "crawlab/constants" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" +) + +func GetAuthQuery(query bson.M, c *gin.Context) bson.M { + user := GetCurrentUser(c) + if user.Role == constants.RoleAdmin { + // 获得所有数据 + return query + } else { + // 只获取自己的数据 + query["user_id"] = user.Id + return query + } +} + diff --git a/backend/services/clean.go b/backend/services/clean.go new file mode 100644 index 00000000..bbd3571d --- /dev/null +++ b/backend/services/clean.go @@ -0,0 +1,122 @@ +package services + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/apex/log" + "github.com/globalsign/mgo/bson" + "runtime/debug" +) + +func InitTaskCleanUserIds() { + adminUser, err := GetAdminUser() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } + tasks, err := model.GetTaskList(nil, 0, constants.Infinite, "+_id") + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } + for _, t := range tasks { + if !t.ScheduleId.Valid() { + t.ScheduleId = bson.ObjectIdHex(constants.ObjectIdNull) + if err := t.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } + + if !t.UserId.Valid() { + t.UserId = adminUser.Id + if err := t.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } + } +} + +func InitProjectCleanUserIds() { + adminUser, err := GetAdminUser() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } + projects, err := model.GetProjectList(nil, "+_id") + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } + for _, p := range projects { + if !p.UserId.Valid() { + p.UserId = adminUser.Id + if err := p.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } + } +} + +func InitSpiderCleanUserIds() { + adminUser, err := GetAdminUser() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } + spiders, _ := model.GetSpiderAllList(nil) + for _, s := range spiders { + if !s.UserId.Valid() { + s.UserId = adminUser.Id + if err := s.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } + } +} + +func InitScheduleCleanUserIds() { + adminUser, err := GetAdminUser() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } + schedules, _ := model.GetScheduleList(nil) + for _, s := range schedules { + if !s.UserId.Valid() { + s.UserId = adminUser.Id + if err := s.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } + } +} + +func InitCleanService() error { + if model.IsMaster() { + // 清理任务UserIds + InitTaskCleanUserIds() + // 清理项目UserIds + InitProjectCleanUserIds() + // 清理爬虫UserIds + InitSpiderCleanUserIds() + // 清理定时任务UserIds + InitScheduleCleanUserIds() + } + return nil +} diff --git a/backend/services/spider.go b/backend/services/spider.go index 0fbc5ebb..f9623316 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -545,6 +545,9 @@ func InitSpiderService() error { if err := GitCron.Start(); err != nil { return err } + + // 清理UserId + InitSpiderCleanUserIds() } return nil diff --git a/backend/services/user.go b/backend/services/user.go index 5e283044..adc56136 100644 --- a/backend/services/user.go +++ b/backend/services/user.go @@ -120,3 +120,11 @@ func GetCurrentUser(c *gin.Context) *model.User { func GetCurrentUserId(c *gin.Context) bson.ObjectId { return GetCurrentUser(c).Id } + +func GetAdminUser() (user *model.User, err error) { + u, err := model.GetUserByUsername("admin") + if err != nil { + return user, err + } + return &u, nil +} diff --git a/frontend/src/components/File/FileDetail.vue b/frontend/src/components/File/FileDetail.vue index ee88cef7..92143019 100644 --- a/frontend/src/components/File/FileDetail.vue +++ b/frontend/src/components/File/FileDetail.vue @@ -1,12 +1,18 @@ + + From 2fa5ff1486c4823ea635014ffee8a51de7f1ab51 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sun, 22 Mar 2020 10:23:31 +0800 Subject: [PATCH 13/42] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=8F=8D=E9=A6=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/i18n/zh.js | 1 + frontend/src/views/feedback/Feedback.vue | 56 +++++++++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index c3bf1e12..80685504 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -317,6 +317,7 @@ export default { 'Feedbacks': '反馈', 'Wechat': '微信', 'Content': '内容', + 'Rating': '评分', // 用户 'Super Admin': '超级管理员', diff --git a/frontend/src/views/feedback/Feedback.vue b/frontend/src/views/feedback/Feedback.vue index ef621235..28d7d053 100644 --- a/frontend/src/views/feedback/Feedback.vue +++ b/frontend/src/views/feedback/Feedback.vue @@ -4,6 +4,22 @@ + + + + + + +
import axios from 'axios' +import { + mapState +} from 'vuex' export default { name: 'Feedback', @@ -69,11 +98,17 @@ export default { form: { email: '', wechat: '', - content: '' + content: '', + rating: 0 }, isLoading: false } }, + computed: { + ...mapState('lang', [ + 'lang' + ]) + }, methods: { submit () { this.$refs['form'].validate(async valid => { @@ -86,6 +121,7 @@ export default { email: this.form.email, wechat: this.form.wechat, content: this.form.content, + rating: this.form.rating, v: sessionStorage.getItem('v') }) if (res && res.data.error) { @@ -95,7 +131,8 @@ export default { this.form = { email: '', wechat: '', - content: '' + content: '', + rating: 0 } this.$message.success(this.$t('Submitted successfully')) } catch (e) { @@ -118,4 +155,19 @@ export default { .actions { text-align: right; } + + .rating >>> .el-form-item__content { + display: flex; + align-items: center; + height: 40px; + } + + .notice { + margin-bottom: 20px; + } + + .notice >>> .el-alert__description { + line-height: 24px; + font-size: 16px; + } From 4ebdf5a5585b4700dca6d4fff445b038e682dd7b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sun, 22 Mar 2020 10:24:45 +0800 Subject: [PATCH 14/42] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=8F=8D=E9=A6=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/feedback/Feedback.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/feedback/Feedback.vue b/frontend/src/views/feedback/Feedback.vue index 28d7d053..ff45c5e3 100644 --- a/frontend/src/views/feedback/Feedback.vue +++ b/frontend/src/views/feedback/Feedback.vue @@ -12,7 +12,7 @@