diff --git a/README-zh.md b/README-zh.md index 1adf3dc7..fabab977 100644 --- a/README-zh.md +++ b/README-zh.md @@ -305,6 +305,9 @@ Crawlab使用起来很方便，也很通用，可以适用于几乎任何主流 + + + ## 社区 & 赞助 diff --git a/README.md b/README.md index 25a16c8e..edd3b6d7 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,9 @@ Crawlab is easy to use, general enough to adapt spiders in any language and any + + + ## Community & Sponsorship diff --git a/backend/main.go b/backend/main.go index 1b568ace..6550bbbd 100644 --- a/backend/main.go +++ b/backend/main.go @@ -265,7 +265,8 @@ func main() { } // 挑战 { - authGroup.GET("/challenges", routes.GetChallengeList) // 挑战列表 + authGroup.GET("/challenges", routes.GetChallengeList) // 挑战列表 + authGroup.POST("/challenges-check", routes.CheckChallengeList) // 检查挑战列表 } // 操作 { diff --git a/backend/mock/spider.go b/backend/mock/spider.go index ef3e6104..e6c20a6b 100644 --- a/backend/mock/spider.go +++ b/backend/mock/spider.go @@ -1,6 +1,7 @@ package mock import ( + "crawlab/constants" "crawlab/model" "github.com/apex/log" "github.com/gin-gonic/gin" @@ -26,6 +27,7 @@ var SpiderList = []model.Spider{ LastRunTs: time.Now(), CreateTs: time.Now(), UpdateTs: time.Now(), + UserId: constants.ObjectIdNull, }, } diff --git a/backend/mock/spider_test.go b/backend/mock/spider_test.go index f4dbea63..11e2c9cd 100644 --- a/backend/mock/spider_test.go +++ b/backend/mock/spider_test.go @@ -2,6 +2,7 @@ package mock import ( "bytes" + "crawlab/constants" "crawlab/model" "encoding/json" "github.com/globalsign/mgo/bson" @@ -61,6 +62,7 @@ func TestPostSpider(t *testing.T) { LastRunTs: time.Now(), CreateTs: time.Now(), UpdateTs: time.Now(), + UserId: constants.ObjectIdNull, } var resp Response var spiderId = "5d429e6c19f7abede924fee2" diff --git a/backend/model/action.go b/backend/model/action.go index 14968cec..15406181 100644 --- a/backend/model/action.go +++ b/backend/model/action.go @@ -1,6 +1,7 @@ package model import ( + "crawlab/constants" "crawlab/database" "github.com/apex/log" "github.com/globalsign/mgo/bson" @@ -82,6 +83,52 @@ func GetActionListTotal(filter interface{}) (int, error) { return result, nil } +func GetVisitDays(uid bson.ObjectId) (int, error) { + type ResData struct { + Days int `json:"days" bson:"days"` + } + s, c := database.GetCol("actions") + defer s.Close() + + pipeline := []bson.M{ + { + "$match": bson.M{ + "user_id": uid, + "type": constants.ActionTypeVisit, + }, + }, + { + "$addFields": bson.M{ + "date": bson.M{ + "$dateToString": bson.M{ + "format": "%Y%m%d", + "date": "$create_ts", + "timezone": "Asia/Shanghai", + }, + }, + }, + }, + { + "$group": bson.M{ + "_id": "$date", + }, + }, + { + "_id": nil, + "days": bson.M{"$sum": 1}, + }, + } + + var resData []ResData + if err := c.Pipe(pipeline).All(&resData); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return 0, err + } + + return resData[0].Days, nil +} + func UpdateAction(id bson.ObjectId, item Action) error { s, c := database.GetCol("actions") defer s.Close() diff --git a/backend/model/challenge.go b/backend/model/challenge.go index 9a96660b..083d216b 100644 --- a/backend/model/challenge.go +++ b/backend/model/challenge.go @@ -97,8 +97,28 @@ func GetChallengeList(filter interface{}, skip int, limit int, sortKey string) ( return challenges, err } - //for _, ch := range challenges { - //} + return challenges, nil +} + +func GetChallengeListWithAchieved(filter interface{}, skip int, limit int, sortKey string, uid bson.ObjectId) ([]Challenge, error) { + challenges, err := GetChallengeList(filter, skip, limit, sortKey) + if err != nil { + return challenges, err + } + + for i, ch := range challenges { + query := bson.M{ + "user_id": uid, + "challenge_id": ch.Id, + } + + list, err := GetChallengeAchievementList(query, 0, 1, "-_id") + if err != nil { + continue + } + + challenges[i].Achieved = len(list) > 0 + } return challenges, nil } @@ -152,3 +172,16 @@ func (ca *ChallengeAchievement) Add() error { return nil } + +func GetChallengeAchievementList(filter interface{}, skip int, limit int, sortKey string) ([]ChallengeAchievement, error) { + s, c := database.GetCol("challenges_achievements") + defer s.Close() + + var challengeAchievements []ChallengeAchievement + if err := c.Find(filter).Skip(skip).Limit(limit).Sort(sortKey).All(&challengeAchievements); err != nil { + debug.PrintStack() + return challengeAchievements, err + } + + return challengeAchievements, nil +} diff --git a/backend/model/project.go b/backend/model/project.go index 92c72655..bc7ca35e 100644 --- a/backend/model/project.go +++ b/backend/model/project.go @@ -15,11 +15,12 @@ type Project struct { Description string `json:"description" bson:"description"` Tags []string `json:"tags" bson:"tags"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` - // 前端展示 Spiders []Spider `json:"spiders" bson:"spiders"` + + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } func (p *Project) Save() error { diff --git a/backend/model/spider.go b/backend/model/spider.go index 49f735a4..5bc2579b 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -65,8 +65,9 @@ type Spider struct { LatestTasks []Task `json:"latest_tasks"` // 最近任务列表 // 时间 - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } // 更新爬虫 diff --git a/backend/model/task.go b/backend/model/task.go index abb5ffc5..6b2a44c1 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -25,14 +25,16 @@ type Task struct { RuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"` TotalDuration float64 `json:"total_duration" bson:"total_duration"` Pid int `json:"pid" bson:"pid"` - UserId bson.ObjectId `json:"user_id" bson:"user_id"` + RunType string `json:"run_type" bson:"run_type"` + ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"` // 前端数据 SpiderName string `json:"spider_name"` NodeName string `json:"node_name"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } type TaskDailyItem struct { diff --git a/backend/routes/action.go b/backend/routes/action.go index 26ffe66e..f0d262ab 100644 --- a/backend/routes/action.go +++ b/backend/routes/action.go @@ -56,7 +56,7 @@ func PutAction(c *gin.Context) { return } - action.UserId = services.GetCurrentUser(c).Id + action.UserId = services.GetCurrentUserId(c) if err := action.Add(); err != nil { HandleError(http.StatusInternalServerError, c, err) diff --git a/backend/routes/challenge.go b/backend/routes/challenge.go index f8d09b6e..1f03654a 100644 --- a/backend/routes/challenge.go +++ b/backend/routes/challenge.go @@ -3,13 +3,15 @@ package routes import ( "crawlab/constants" "crawlab/model" + "crawlab/services" + "crawlab/services/challenge" "github.com/gin-gonic/gin" "net/http" ) func GetChallengeList(c *gin.Context) { // 获取列表 - users, err := model.GetChallengeList(nil, 0, constants.Infinite, "create_ts") + users, err := model.GetChallengeListWithAchieved(nil, 0, constants.Infinite, "create_ts", services.GetCurrentUserId(c)) if err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -29,3 +31,15 @@ func GetChallengeList(c *gin.Context) { Total: total, }) } + +func CheckChallengeList(c *gin.Context) { + uid := services.GetCurrentUserId(c) + if err := challenge.CheckChallengeAndUpdateAll(uid); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index 3776019c..601d467f 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -77,7 +77,7 @@ func PutSchedule(c *gin.Context) { } // 加入用户ID - item.UserId = services.GetCurrentUser(c).Id + item.UserId = services.GetCurrentUserId(c) // 更新数据库 if err := model.AddSchedule(item); err != nil { diff --git a/backend/routes/spider.go b/backend/routes/spider.go index d195a5e7..b136070b 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -189,6 +189,9 @@ func PutSpider(c *gin.Context) { // 将FileId置空 spider.FileId = bson.ObjectIdHex(constants.ObjectIdNull) + // UserId + spider.UserId = services.GetCurrentUserId(c) + // 爬虫目录 spiderDir := filepath.Join(viper.GetString("spider.path"), spider.Name) @@ -274,6 +277,9 @@ func CopySpider(c *gin.Context) { return } + // UserId + spider.UserId = services.GetCurrentUserId(c) + // 复制爬虫 if err := services.CopySpider(spider, reqBody.Name); err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -365,6 +371,8 @@ func UploadSpider(c *gin.Context) { Type: constants.Customized, Src: filepath.Join(srcPath, spiderName), FileId: fid, + ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), + UserId: services.GetCurrentUserId(c), } if name != "" { spider.Name = name @@ -617,7 +625,8 @@ func RunSelectedSpider(c *gin.Context) { SpiderId: taskParam.SpiderId, NodeId: node.Id, Param: taskParam.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeAllNodes, } id, err := services.AddTask(t) @@ -633,7 +642,8 @@ func RunSelectedSpider(c *gin.Context) { t := model.Task{ SpiderId: taskParam.SpiderId, Param: taskParam.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeRandom, } id, err := services.AddTask(t) if err != nil { @@ -648,7 +658,8 @@ func RunSelectedSpider(c *gin.Context) { SpiderId: taskParam.SpiderId, NodeId: nodeId, Param: taskParam.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeSelectedNodes, } id, err := services.AddTask(t) diff --git a/backend/routes/task.go b/backend/routes/task.go index 2880abb9..ecfbc71f 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -115,7 +115,8 @@ func PutTask(c *gin.Context) { SpiderId: reqBody.SpiderId, NodeId: node.Id, Param: reqBody.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeAllNodes, } id, err := services.AddTask(t) @@ -131,7 +132,8 @@ func PutTask(c *gin.Context) { t := model.Task{ SpiderId: reqBody.SpiderId, Param: reqBody.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeRandom, } id, err := services.AddTask(t) if err != nil { @@ -146,7 +148,8 @@ func PutTask(c *gin.Context) { SpiderId: reqBody.SpiderId, NodeId: nodeId, Param: reqBody.Param, - UserId: services.GetCurrentUser(c).Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeSelectedNodes, } id, err := services.AddTask(t) diff --git a/backend/services/challenge/base.go b/backend/services/challenge/base.go index 6529d887..9515ff9e 100644 --- a/backend/services/challenge/base.go +++ b/backend/services/challenge/base.go @@ -15,14 +15,14 @@ type Service interface { Check() (bool, error) } -func GetService(name string) Service { +func GetService(name string, uid bson.ObjectId) Service { switch name { case constants.ChallengeLogin7d: - return &Login7dService{} + return &Login7dService{UserId: uid} case constants.ChallengeCreateCustomizedSpider: - return &CreateCustomizedSpiderService{} + return &CreateCustomizedSpiderService{UserId: uid} case constants.ChallengeRunRandom: - return &RunRandomService{} + return &RunRandomService{UserId: uid} } return nil } @@ -42,14 +42,14 @@ func AddChallengeAchievement(name string, uid bson.ObjectId) error { return nil } -func CheckChallengeAndUpdate(name string, uid bson.ObjectId) error { - svc := GetService(name) +func CheckChallengeAndUpdate(ch model.Challenge, uid bson.ObjectId) error { + svc := GetService(ch.Name, uid) achieved, err := svc.Check() if err != nil { return err } - if achieved { - if err := AddChallengeAchievement(name, uid); err != nil { + if achieved && !ch.Achieved { + if err := AddChallengeAchievement(ch.Name, uid); err != nil { return err } } @@ -57,6 +57,15 @@ func CheckChallengeAndUpdate(name string, uid bson.ObjectId) error { } func CheckChallengeAndUpdateAll(uid bson.ObjectId) error { + challenges, err := model.GetChallengeListWithAchieved(nil, 0, constants.Infinite, "-_id", uid) + if err != nil { + return err + } + for _, ch := range challenges { + if err := CheckChallengeAndUpdate(ch, uid); err != nil { + continue + } + } return nil } diff --git a/backend/services/challenge/create_customized_spider.go b/backend/services/challenge/create_customized_spider.go index cad1aa25..6c61318f 100644 --- a/backend/services/challenge/create_customized_spider.go +++ b/backend/services/challenge/create_customized_spider.go @@ -1,8 +1,23 @@ package challenge +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + type CreateCustomizedSpiderService struct { + UserId bson.ObjectId } func (s *CreateCustomizedSpiderService) Check() (bool, error) { - return true, nil + query := bson.M{ + "user_id": s.UserId, + "type": constants.Customized, + } + _, count, err := model.GetSpiderList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return count > 0, nil } diff --git a/backend/services/challenge/login_7d.go b/backend/services/challenge/login_7d.go index c10e2d38..91540423 100644 --- a/backend/services/challenge/login_7d.go +++ b/backend/services/challenge/login_7d.go @@ -1,8 +1,18 @@ package challenge +import ( + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + type Login7dService struct { + UserId bson.ObjectId } func (s *Login7dService) Check() (bool, error) { - return true, nil + days, err := model.GetVisitDays(s.UserId) + if err != nil { + return false, err + } + return days >= 7, nil } diff --git a/backend/services/challenge/run_random.go b/backend/services/challenge/run_random.go index dd9e4a40..ebdc11fe 100644 --- a/backend/services/challenge/run_random.go +++ b/backend/services/challenge/run_random.go @@ -1,8 +1,23 @@ package challenge +import ( + "crawlab/constants" + "crawlab/model" + "github.com/globalsign/mgo/bson" +) + type RunRandomService struct { + UserId bson.ObjectId } func (s *RunRandomService) Check() (bool, error) { - return true, nil + query := bson.M{ + "user_id": s.UserId, + "run_type": constants.RunTypeRandom, + } + list, err := model.GetTaskList(query, 0, 1, "-_id") + if err != nil { + return false, err + } + return len(list) > 0, nil } diff --git a/backend/services/spider.go b/backend/services/spider.go index 77bb19ab..0fbc5ebb 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -471,6 +471,7 @@ func InitDemoSpiders () { ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), FileId: bson.ObjectIdHex(constants.ObjectIdNull), Cmd: configData.Cmd, + UserId: bson.ObjectIdHex(constants.ObjectIdNull), } if err := spider.Add(); err != nil { log.Errorf("add spider error: " + err.Error()) @@ -497,6 +498,7 @@ func InitDemoSpiders () { ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), FileId: bson.ObjectIdHex(constants.ObjectIdNull), Config: configData, + UserId: bson.ObjectIdHex(constants.ObjectIdNull), } if err := spider.Add(); err != nil { log.Errorf("add spider error: " + err.Error()) diff --git a/backend/services/user.go b/backend/services/user.go index a01e721b..b01b4923 100644 --- a/backend/services/user.go +++ b/backend/services/user.go @@ -115,3 +115,8 @@ func GetCurrentUser(c *gin.Context) *model.User { data, _ := c.Get("currentUser") return data.(*model.User) } + +func GetCurrentUserId(c *gin.Context) bson.ObjectId { + return GetCurrentUser(c).Id +} + diff --git a/frontend/src/views/challenge/ChallengeList.vue b/frontend/src/views/challenge/ChallengeList.vue index c0bda6f1..3d45bd35 100644 --- a/frontend/src/views/challenge/ChallengeList.vue +++ b/frontend/src/views/challenge/ChallengeList.vue @@ -77,6 +77,7 @@ export default { }, methods: { async getData () { + await this.$request.post('/challenges-check') const res = await this.$request.get('/challenges') this.challenges = res.data.data || [] }