diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index 4099e43a..02c903ce 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,3 +1,9 @@ +# 0.4.9 (TBC) +### 功能 / 优化 + +### Bug 修复 +- **CLI 无法在 Windows 上使用**. [#580](https://github.com/crawlab-team/crawlab/issues/580) + # 0.4.8 (2020-03-11) ### 功能 / 优化 - **支持更多编程语言安装**. 现在用户可以安装或预装更多的编程语言，包括 Java、.Net Core、PHP. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b4792f3..d5e7c72e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 0.4.9 (TBC) +### Features / Enhancement + +### Bug Fixes +- **CLI unable to use on Windows**. [#580](https://github.com/crawlab-team/crawlab/issues/580) + # 0.4.8 (2020-03-11) ### Features / Enhancement - **Support Installations of More Programming Languages**. Now users can install or pre-install more programming languages including Java, .Net Core and PHP. diff --git a/backend/constants/action.go b/backend/constants/action.go new file mode 100644 index 00000000..31d3655e --- /dev/null +++ b/backend/constants/action.go @@ -0,0 +1,5 @@ +package constants + +const ( + ActionTypeVisit = "visit" +) diff --git a/backend/constants/challenge.go b/backend/constants/challenge.go new file mode 100644 index 00000000..4cfa9710 --- /dev/null +++ b/backend/constants/challenge.go @@ -0,0 +1,7 @@ +package constants + +const ( + ChallengeLogin7d = "login_7d" + ChallengeCreateCustomizedSpider = "create_customized_spider" + ChallengeRunRandom = "run_random" +) diff --git a/backend/data/challenge_data.json b/backend/data/challenge_data.json new file mode 100644 index 00000000..e8cea7da --- /dev/null +++ b/backend/data/challenge_data.json @@ -0,0 +1,28 @@ +[ + { + "name": "login_7d", + "title_cn": "连续登录 7 天", + "title_en": "Logged-in for 7 days", + "description_cn": "连续 7 天登录 Crawlab，即可完成挑战！", + "description_en": "Logged-in for consecutive 7 days to complete the challenge", + "difficulty": 1 + }, + { + "name": "create_customized_spider", + "title_cn": "创建一个自定义爬虫", + "title_en": "Create a customized spider", + "description_cn": "在爬虫列表中，点击 '添加爬虫'，选择 '自定义爬虫'，输入相应的参数，点击添加，即可完成挑战！", + "description_en": "In Spider List page, click 'Add Spider', select 'Customized Spider', enter params, click 'Add' to finish the challenge.", + "difficulty": 1, + "path": "/spiders" + }, + { + "name": "run_random", + "title_cn": "用随机模式运行一个爬虫", + "title_en": "Run a spider in random mode", + "description_cn": "在您创建好的爬虫中，导航到其对应的详情页（爬虫列表中点击爬虫），运行一个爬虫，选择随机模式。", + "description_en": "In your created spiders, navigate to corresponding detail page (click spider in Spider List page), run a spider in random mode.", + "difficulty": 2, + "path": "/spiders" + } +] \ No newline at end of file diff --git a/backend/entity/rpc.go b/backend/entity/rpc.go index 3f5ddcea..48f14b26 100644 --- a/backend/entity/rpc.go +++ b/backend/entity/rpc.go @@ -1,11 +1,11 @@ package entity type RpcMessage struct { - Id string `json:"id"` - Method string `json:"method"` - NodeId string `json:"node_id"` - Params map[string]string `json:"params"` - Timeout int `json:"timeout"` - Result string `json:"result"` - Error string `json:"error"` + Id string `json:"id"` // 消息ID + Method string `json:"method"` // 消息方法 + NodeId string `json:"node_id"` // 节点ID + Params map[string]string `json:"params"` // 参数 + Timeout int `json:"timeout"` // 超时 + Result string `json:"result"` // 结果 + Error string `json:"error"` // 错误 } diff --git a/backend/main.go b/backend/main.go index d53991e2..1b568ace 100644 --- a/backend/main.go +++ b/backend/main.go @@ -9,6 +9,7 @@ import ( "crawlab/model" "crawlab/routes" "crawlab/services" + "crawlab/services/challenge" "crawlab/services/rpc" "github.com/apex/log" "github.com/gin-gonic/gin" @@ -91,6 +92,14 @@ func main() { panic(err) } log.Info("initialized dependency fetcher successfully") + + // 初始化挑战服务 + if err := challenge.InitChallengeService(); err != nil { + log.Error("init challenge service error:" + err.Error()) + debug.PrintStack() + panic(err) + } + log.Info("initialized challenge service successfully") } // 初始化任务执行器 @@ -254,6 +263,17 @@ func main() { authGroup.POST("/projects/:id", routes.PostProject) // 新增 authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除 } + // 挑战 + { + authGroup.GET("/challenges", routes.GetChallengeList) // 挑战列表 + } + // 操作 + { + //authGroup.GET("/actions", routes.GetActionList) // 操作列表 + //authGroup.GET("/actions/:id", routes.GetAction) // 操作 + authGroup.PUT("/actions", routes.PutAction) // 新增操作 + //authGroup.POST("/actions/:id", routes.PostAction) // 修改操作 + } // 统计数据 authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 // 文件 @@ -262,7 +282,7 @@ func main() { authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支 authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits - authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits + authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits } } diff --git a/backend/model/action.go b/backend/model/action.go new file mode 100644 index 00000000..14968cec --- /dev/null +++ b/backend/model/action.go @@ -0,0 +1,115 @@ +package model + +import ( + "crawlab/database" + "github.com/apex/log" + "github.com/globalsign/mgo/bson" + "runtime/debug" + "time" +) + +type Action struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + Type string `json:"type" bson:"type"` + + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` +} + +func (a *Action) Save() error { + s, c := database.GetCol("actions") + defer s.Close() + + a.UpdateTs = time.Now() + + if err := c.UpdateId(a.Id, a); err != nil { + debug.PrintStack() + return err + } + return nil +} + +func (a *Action) Add() error { + s, c := database.GetCol("actions") + defer s.Close() + + a.Id = bson.NewObjectId() + a.UpdateTs = time.Now() + a.CreateTs = time.Now() + if err := c.Insert(a); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +func GetAction(id bson.ObjectId) (Action, error) { + s, c := database.GetCol("actions") + defer s.Close() + var user Action + if err := c.Find(bson.M{"_id": id}).One(&user); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return user, err + } + return user, nil +} + +func GetActionList(filter interface{}, skip int, limit int, sortKey string) ([]Action, error) { + s, c := database.GetCol("actions") + defer s.Close() + + var actions []Action + if err := c.Find(filter).Skip(skip).Limit(limit).Sort(sortKey).All(&actions); err != nil { + debug.PrintStack() + return actions, err + } + return actions, nil +} + +func GetActionListTotal(filter interface{}) (int, error) { + s, c := database.GetCol("actions") + defer s.Close() + + var result int + result, err := c.Find(filter).Count() + if err != nil { + return result, err + } + return result, nil +} + +func UpdateAction(id bson.ObjectId, item Action) error { + s, c := database.GetCol("actions") + defer s.Close() + + var result Action + if err := c.FindId(id).One(&result); err != nil { + debug.PrintStack() + return err + } + + if err := item.Save(); err != nil { + return err + } + return nil +} + +func RemoveAction(id bson.ObjectId) error { + s, c := database.GetCol("actions") + defer s.Close() + + var result Action + if err := c.FindId(id).One(&result); err != nil { + return err + } + + if err := c.RemoveId(id); err != nil { + return err + } + + return nil +} diff --git a/backend/model/challenge.go b/backend/model/challenge.go new file mode 100644 index 00000000..9a96660b --- /dev/null +++ b/backend/model/challenge.go @@ -0,0 +1,154 @@ +package model + +import ( + "crawlab/database" + "github.com/apex/log" + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" + "runtime/debug" + "time" +) + +type Challenge struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + TitleCn string `json:"title_cn" bson:"title_cn"` + TitleEn string `json:"title_en" bson:"title_en"` + DescriptionCn string `json:"description_cn" bson:"description_cn"` + DescriptionEn string `json:"description_en" bson:"description_en"` + Difficulty int `json:"difficulty" bson:"difficulty"` + Path string `json:"path" bson:"path"` + + // 前端展示 + Achieved bool `json:"achieved" bson:"achieved"` + + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` +} + +func (ch *Challenge) Save() error { + s, c := database.GetCol("challenges") + defer s.Close() + + ch.UpdateTs = time.Now() + + if err := c.UpdateId(ch.Id, c); err != nil { + debug.PrintStack() + return err + } + return nil +} + +func (ch *Challenge) Add() error { + s, c := database.GetCol("challenges") + defer s.Close() + + ch.Id = bson.NewObjectId() + ch.UpdateTs = time.Now() + ch.CreateTs = time.Now() + if err := c.Insert(ch); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +func GetChallenge(id bson.ObjectId) (Challenge, error) { + s, c := database.GetCol("challenges") + defer s.Close() + + var ch Challenge + if err := c.Find(bson.M{"_id": id}).One(&ch); err != nil { + if err != mgo.ErrNotFound { + log.Errorf(err.Error()) + debug.PrintStack() + return ch, err + } + } + + return ch, nil +} + +func GetChallengeByName(name string) (Challenge, error) { + s, c := database.GetCol("challenges") + defer s.Close() + + var ch Challenge + if err := c.Find(bson.M{"name": name}).One(&ch); err != nil { + if err != mgo.ErrNotFound { + log.Errorf(err.Error()) + debug.PrintStack() + return ch, err + } + } + + return ch, nil +} + +func GetChallengeList(filter interface{}, skip int, limit int, sortKey string) ([]Challenge, error) { + s, c := database.GetCol("challenges") + defer s.Close() + + var challenges []Challenge + if err := c.Find(filter).Skip(skip).Limit(limit).Sort(sortKey).All(&challenges); err != nil { + debug.PrintStack() + return challenges, err + } + + //for _, ch := range challenges { + //} + + return challenges, nil +} + +func GetChallengeListTotal(filter interface{}) (int, error) { + s, c := database.GetCol("challenges") + defer s.Close() + + var result int + result, err := c.Find(filter).Count() + if err != nil { + return result, err + } + return result, nil +} + +type ChallengeAchievement struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + ChallengeId bson.ObjectId `json:"challenge_id" bson:"challenge_id"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` +} + +func (ca *ChallengeAchievement) Save() error { + s, c := database.GetCol("challenges_achievements") + defer s.Close() + + ca.UpdateTs = time.Now() + + if err := c.UpdateId(ca.Id, c); err != nil { + debug.PrintStack() + return err + } + return nil +} + +func (ca *ChallengeAchievement) Add() error { + s, c := database.GetCol("challenges_achievements") + defer s.Close() + + ca.Id = bson.NewObjectId() + ca.UpdateTs = time.Now() + ca.CreateTs = time.Now() + if err := c.Insert(ca); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} diff --git a/backend/routes/action.go b/backend/routes/action.go new file mode 100644 index 00000000..26ffe66e --- /dev/null +++ b/backend/routes/action.go @@ -0,0 +1,114 @@ +package routes + +import ( + "crawlab/model" + "crawlab/services" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" +) + +func GetAction(c *gin.Context) { + id := c.Param("id") + + user, err := model.GetAction(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: user, + }) +} + +func GetActionList(c *gin.Context) { + pageNum := c.GetInt("page_num") + pageSize := c.GetInt("page_size") + + users, err := model.GetActionList(nil, (pageNum-1)*pageSize, pageSize, "-create_ts") + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + total, err := model.GetActionListTotal(nil) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, ListResponse{ + Status: "ok", + Message: "success", + Data: users, + Total: total, + }) +} + +func PutAction(c *gin.Context) { + // 绑定请求数据 + var action model.Action + if err := c.ShouldBindJSON(&action); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + action.UserId = services.GetCurrentUser(c).Id + + if err := action.Add(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func PostAction(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + } + + var item model.Action + if err := c.ShouldBindJSON(&item); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + if err := model.UpdateAction(bson.ObjectIdHex(id), item); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func DeleteAction(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } + + // 从数据库中删除该爬虫 + if err := model.RemoveAction(bson.ObjectIdHex(id)); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/routes/challenge.go b/backend/routes/challenge.go new file mode 100644 index 00000000..f8d09b6e --- /dev/null +++ b/backend/routes/challenge.go @@ -0,0 +1,31 @@ +package routes + +import ( + "crawlab/constants" + "crawlab/model" + "github.com/gin-gonic/gin" + "net/http" +) + +func GetChallengeList(c *gin.Context) { + // 获取列表 + users, err := model.GetChallengeList(nil, 0, constants.Infinite, "create_ts") + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取总数 + total, err := model.GetChallengeListTotal(nil) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, ListResponse{ + Status: "ok", + Message: "success", + Data: users, + Total: total, + }) +} diff --git a/backend/services/challenge/base.go b/backend/services/challenge/base.go new file mode 100644 index 00000000..6529d887 --- /dev/null +++ b/backend/services/challenge/base.go @@ -0,0 +1,101 @@ +package challenge + +import ( + "crawlab/constants" + "crawlab/model" + "encoding/json" + "github.com/apex/log" + "github.com/globalsign/mgo/bson" + "io/ioutil" + "path" + "runtime/debug" +) + +type Service interface { + Check() (bool, error) +} + +func GetService(name string) Service { + switch name { + case constants.ChallengeLogin7d: + return &Login7dService{} + case constants.ChallengeCreateCustomizedSpider: + return &CreateCustomizedSpiderService{} + case constants.ChallengeRunRandom: + return &RunRandomService{} + } + return nil +} + +func AddChallengeAchievement(name string, uid bson.ObjectId) error { + ch, err := model.GetChallengeByName(name) + if err != nil { + return err + } + ca := model.ChallengeAchievement{ + ChallengeId: ch.Id, + UserId: uid, + } + if err := ca.Add(); err != nil { + return err + } + return nil +} + +func CheckChallengeAndUpdate(name string, uid bson.ObjectId) error { + svc := GetService(name) + achieved, err := svc.Check() + if err != nil { + return err + } + if achieved { + if err := AddChallengeAchievement(name, uid); err != nil { + return err + } + } + return nil +} + +func CheckChallengeAndUpdateAll(uid bson.ObjectId) error { + return nil +} + +func InitChallengeService() error { + // 读取文件 + contentBytes, err := ioutil.ReadFile(path.Join("data", "challenge_data.json")) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 反序列化 + var challenges []model.Challenge + if err := json.Unmarshal(contentBytes, &challenges); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + for _, ch := range challenges { + chDb, err := model.GetChallengeByName(ch.Name) + if err != nil { + continue + } + if chDb.Name == "" { + if err := ch.Add(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } else { + if err := ch.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + } + } + + return nil +} diff --git a/backend/services/challenge/create_customized_spider.go b/backend/services/challenge/create_customized_spider.go new file mode 100644 index 00000000..cad1aa25 --- /dev/null +++ b/backend/services/challenge/create_customized_spider.go @@ -0,0 +1,8 @@ +package challenge + +type CreateCustomizedSpiderService struct { +} + +func (s *CreateCustomizedSpiderService) Check() (bool, error) { + return true, nil +} diff --git a/backend/services/challenge/login_7d.go b/backend/services/challenge/login_7d.go new file mode 100644 index 00000000..c10e2d38 --- /dev/null +++ b/backend/services/challenge/login_7d.go @@ -0,0 +1,8 @@ +package challenge + +type Login7dService struct { +} + +func (s *Login7dService) Check() (bool, error) { + return true, nil +} diff --git a/backend/services/challenge/run_random.go b/backend/services/challenge/run_random.go new file mode 100644 index 00000000..dd9e4a40 --- /dev/null +++ b/backend/services/challenge/run_random.go @@ -0,0 +1,8 @@ +package challenge + +type RunRandomService struct { +} + +func (s *RunRandomService) Check() (bool, error) { + return true, nil +} diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 93d41827..1917a490 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -51,6 +51,11 @@ export default { // remove loading-placeholder const elLoading = document.querySelector('#loading-placeholder') elLoading.remove() + + // send visit event + await this.$request.put('/actions', { + type: 'visit' + }) } } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 899d9d44..e677d90b 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -459,6 +459,14 @@ export default { 'General': '通用', 'Enable Tutorial': '启用教程', + // 挑战 + 'Challenge': '挑战', + 'Challenges': '挑战', + 'Difficulty': '难度', + 'Achieved': '已达成', + 'Not Achieved': '未达成', + 'Start Challenge': '开始挑战', + // 全局 'Related Documentation': '相关文档', 'Click to view related Documentation': '点击查看相关文档', diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 69b7f35b..276fe49d 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -197,7 +197,7 @@ export const constantRouterMap = [ component: Layout, meta: { title: 'User', - icon: 'fa fa-user' + icon: 'fa fa-users' }, children: [ { @@ -206,7 +206,26 @@ export const constantRouterMap = [ component: () => import('../views/user/UserList'), meta: { title: 'Users', - icon: 'fa fa-user' + icon: 'fa fa-users' + } + } + ] + }, + { + path: '/challenges', + component: Layout, + meta: { + title: 'User', + icon: 'fa fa-flash' + }, + children: [ + { + path: '', + name: 'ChallengeList', + component: () => import('../views/challenge/ChallengeList'), + meta: { + title: 'Challenges', + icon: 'fa fa-flash' } } ] diff --git a/frontend/src/views/challenge/ChallengeList.vue b/frontend/src/views/challenge/ChallengeList.vue new file mode 100644 index 00000000..c0bda6f1 --- /dev/null +++ b/frontend/src/views/challenge/ChallengeList.vue @@ -0,0 +1,188 @@ + + + + +