git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

updated contributors

Browse Source
This commit is contained in:
marvzhang
2020-03-19 11:13:25 +08:00
parent a88e90e025
commit e3a059eb7c
22 changed files with 211 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
README-zh.md
View File

@@ -305,6 +305,9 @@ Crawlab使用起来很方便，也很通用，可以适用于几乎任何主流
<a href="https://github.com/duanbin0414">
<img src="https://avatars3.githubusercontent.com/u/50389867?s=460&v=4" height="80">
</a>
<a href="https://github.com/zkqiang">
<img src="https://avatars3.githubusercontent.com/u/32983588?s=460&u=83082ddc0a3020279374b94cce70f1aebb220b3d&v=4" height="80">
</a>
## 社区 & 赞助

3
README.md
View File

@@ -272,6 +272,9 @@ Crawlab is easy to use, general enough to adapt spiders in any language and any
<a href="https://github.com/duanbin0414">
<img src="https://avatars3.githubusercontent.com/u/50389867?s=460&v=4" height="80">
</a>
<a href="https://github.com/zkqiang">
<img src="https://avatars3.githubusercontent.com/u/32983588?s=460&u=83082ddc0a3020279374b94cce70f1aebb220b3d&v=4" height="80">
</a>
## Community & Sponsorship

3
backend/main.go
View File

@@ -265,7 +265,8 @@ func main() {
}
// 挑战
{
authGroup.GET("/challenges", routes.GetChallengeList) // 挑战列表
authGroup.GET("/challenges", routes.GetChallengeList) // 挑战列表
authGroup.POST("/challenges-check", routes.CheckChallengeList) // 检查挑战列表
}
// 操作
{

2
backend/mock/spider.go
View File

@@ -1,6 +1,7 @@
package mock
import (
"crawlab/constants"
"crawlab/model"
"github.com/apex/log"
"github.com/gin-gonic/gin"
@@ -26,6 +27,7 @@ var SpiderList = []model.Spider{
LastRunTs: time.Now(),
CreateTs: time.Now(),
UpdateTs: time.Now(),
UserId: constants.ObjectIdNull,
},
}

2
backend/mock/spider_test.go
View File

@@ -2,6 +2,7 @@ package mock
import (
"bytes"
"crawlab/constants"
"crawlab/model"
"encoding/json"
"github.com/globalsign/mgo/bson"
@@ -61,6 +62,7 @@ func TestPostSpider(t *testing.T) {
LastRunTs: time.Now(),
CreateTs: time.Now(),
UpdateTs: time.Now(),
UserId: constants.ObjectIdNull,
}
var resp Response
var spiderId = "5d429e6c19f7abede924fee2"

47
backend/model/action.go
View File

@@ -1,6 +1,7 @@
package model
import (
"crawlab/constants"
"crawlab/database"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
@@ -82,6 +83,52 @@ func GetActionListTotal(filter interface{}) (int, error) {
return result, nil
}
func GetVisitDays(uid bson.ObjectId) (int, error) {
type ResData struct {
Days int `json:"days" bson:"days"`
}
s, c := database.GetCol("actions")
defer s.Close()
pipeline := []bson.M{
{
"$match": bson.M{
"user_id": uid,
"type": constants.ActionTypeVisit,
},
},
{
"$addFields": bson.M{
"date": bson.M{
"$dateToString": bson.M{
"format": "%Y%m%d",
"date": "$create_ts",
"timezone": "Asia/Shanghai",
},
},
},
},
{
"$group": bson.M{
"_id": "$date",
},
},
{
"_id": nil,
"days": bson.M{"$sum": 1},
},
}
var resData []ResData
if err := c.Pipe(pipeline).All(&resData); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return 0, err
}
return resData[0].Days, nil
}
func UpdateAction(id bson.ObjectId, item Action) error {
s, c := database.GetCol("actions")
defer s.Close()

37
backend/model/challenge.go
View File

@@ -97,8 +97,28 @@ func GetChallengeList(filter interface{}, skip int, limit int, sortKey string) (
return challenges, err
}
//for _, ch := range challenges {
//}
return challenges, nil
}
func GetChallengeListWithAchieved(filter interface{}, skip int, limit int, sortKey string, uid bson.ObjectId) ([]Challenge, error) {
challenges, err := GetChallengeList(filter, skip, limit, sortKey)
if err != nil {
return challenges, err
}
for i, ch := range challenges {
query := bson.M{
"user_id": uid,
"challenge_id": ch.Id,
}
list, err := GetChallengeAchievementList(query, 0, 1, "-_id")
if err != nil {
continue
}
challenges[i].Achieved = len(list) > 0
}
return challenges, nil
}
@@ -152,3 +172,16 @@ func (ca *ChallengeAchievement) Add() error {
return nil
}
func GetChallengeAchievementList(filter interface{}, skip int, limit int, sortKey string) ([]ChallengeAchievement, error) {
s, c := database.GetCol("challenges_achievements")
defer s.Close()
var challengeAchievements []ChallengeAchievement
if err := c.Find(filter).Skip(skip).Limit(limit).Sort(sortKey).All(&challengeAchievements); err != nil {
debug.PrintStack()
return challengeAchievements, err
}
return challengeAchievements, nil
}

7
backend/model/project.go
View File

@@ -15,11 +15,12 @@ type Project struct {
Description string `json:"description" bson:"description"`
Tags []string `json:"tags" bson:"tags"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
// 前端展示
Spiders []Spider `json:"spiders" bson:"spiders"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
}
func (p *Project) Save() error {

5
backend/model/spider.go
View File

@@ -65,8 +65,9 @@ type Spider struct {
LatestTasks []Task `json:"latest_tasks"` // 最近任务列表
// 时间
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
}
// 更新爬虫

8
backend/model/task.go
View File

@@ -25,14 +25,16 @@ type Task struct {
RuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"`
TotalDuration float64 `json:"total_duration" bson:"total_duration"`
Pid int `json:"pid" bson:"pid"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
RunType string `json:"run_type" bson:"run_type"`
ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"`
// 前端数据
SpiderName string `json:"spider_name"`
NodeName string `json:"node_name"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
}
type TaskDailyItem struct {

2
backend/routes/action.go
View File

@@ -56,7 +56,7 @@ func PutAction(c *gin.Context) {
return
}
action.UserId = services.GetCurrentUser(c).Id
action.UserId = services.GetCurrentUserId(c)
if err := action.Add(); err != nil {
HandleError(http.StatusInternalServerError, c, err)

16
backend/routes/challenge.go
View File

@@ -3,13 +3,15 @@ package routes
import (
"crawlab/constants"
"crawlab/model"
"crawlab/services"
"crawlab/services/challenge"
"github.com/gin-gonic/gin"
"net/http"
)
func GetChallengeList(c *gin.Context) {
// 获取列表
users, err := model.GetChallengeList(nil, 0, constants.Infinite, "create_ts")
users, err := model.GetChallengeListWithAchieved(nil, 0, constants.Infinite, "create_ts", services.GetCurrentUserId(c))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
@@ -29,3 +31,15 @@ func GetChallengeList(c *gin.Context) {
Total: total,
})
}
func CheckChallengeList(c *gin.Context) {
uid := services.GetCurrentUserId(c)
if err := challenge.CheckChallengeAndUpdateAll(uid); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}

2
backend/routes/schedule.go
View File

@@ -77,7 +77,7 @@ func PutSchedule(c *gin.Context) {
}
// 加入用户ID
item.UserId = services.GetCurrentUser(c).Id
item.UserId = services.GetCurrentUserId(c)
// 更新数据库
if err := model.AddSchedule(item); err != nil {

17
backend/routes/spider.go
View File

@@ -189,6 +189,9 @@ func PutSpider(c *gin.Context) {
// 将FileId置空
spider.FileId = bson.ObjectIdHex(constants.ObjectIdNull)
// UserId
spider.UserId = services.GetCurrentUserId(c)
// 爬虫目录
spiderDir := filepath.Join(viper.GetString("spider.path"), spider.Name)
@@ -274,6 +277,9 @@ func CopySpider(c *gin.Context) {
return
}
// UserId
spider.UserId = services.GetCurrentUserId(c)
// 复制爬虫
if err := services.CopySpider(spider, reqBody.Name); err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -365,6 +371,8 @@ func UploadSpider(c *gin.Context) {
Type: constants.Customized,
Src: filepath.Join(srcPath, spiderName),
FileId: fid,
ProjectId: bson.ObjectIdHex(constants.ObjectIdNull),
UserId: services.GetCurrentUserId(c),
}
if name != "" {
spider.Name = name
@@ -617,7 +625,8 @@ func RunSelectedSpider(c *gin.Context) {
SpiderId: taskParam.SpiderId,
NodeId: node.Id,
Param: taskParam.Param,
UserId: services.GetCurrentUser(c).Id,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeAllNodes,
}
id, err := services.AddTask(t)
@@ -633,7 +642,8 @@ func RunSelectedSpider(c *gin.Context) {
t := model.Task{
SpiderId: taskParam.SpiderId,
Param: taskParam.Param,
UserId: services.GetCurrentUser(c).Id,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeRandom,
}
id, err := services.AddTask(t)
if err != nil {
@@ -648,7 +658,8 @@ func RunSelectedSpider(c *gin.Context) {
SpiderId: taskParam.SpiderId,
NodeId: nodeId,
Param: taskParam.Param,
UserId: services.GetCurrentUser(c).Id,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeSelectedNodes,
}
id, err := services.AddTask(t)

9
backend/routes/task.go
View File

@@ -115,7 +115,8 @@ func PutTask(c *gin.Context) {
SpiderId: reqBody.SpiderId,
NodeId: node.Id,
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeAllNodes,
}
id, err := services.AddTask(t)
@@ -131,7 +132,8 @@ func PutTask(c *gin.Context) {
t := model.Task{
SpiderId: reqBody.SpiderId,
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeRandom,
}
id, err := services.AddTask(t)
if err != nil {
@@ -146,7 +148,8 @@ func PutTask(c *gin.Context) {
SpiderId: reqBody.SpiderId,
NodeId: nodeId,
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
UserId: services.GetCurrentUserId(c),
RunType: constants.RunTypeSelectedNodes,
}
id, err := services.AddTask(t)

25
backend/services/challenge/base.go
View File

@@ -15,14 +15,14 @@ type Service interface {
Check() (bool, error)
}
func GetService(name string) Service {
func GetService(name string, uid bson.ObjectId) Service {
switch name {
case constants.ChallengeLogin7d:
return &Login7dService{}
return &Login7dService{UserId: uid}
case constants.ChallengeCreateCustomizedSpider:
return &CreateCustomizedSpiderService{}
return &CreateCustomizedSpiderService{UserId: uid}
case constants.ChallengeRunRandom:
return &RunRandomService{}
return &RunRandomService{UserId: uid}
}
return nil
}
@@ -42,14 +42,14 @@ func AddChallengeAchievement(name string, uid bson.ObjectId) error {
return nil
}
func CheckChallengeAndUpdate(name string, uid bson.ObjectId) error {
svc := GetService(name)
func CheckChallengeAndUpdate(ch model.Challenge, uid bson.ObjectId) error {
svc := GetService(ch.Name, uid)
achieved, err := svc.Check()
if err != nil {
return err
}
if achieved {
if err := AddChallengeAchievement(name, uid); err != nil {
if achieved && !ch.Achieved {
if err := AddChallengeAchievement(ch.Name, uid); err != nil {
return err
}
}
@@ -57,6 +57,15 @@ func CheckChallengeAndUpdate(name string, uid bson.ObjectId) error {
}
func CheckChallengeAndUpdateAll(uid bson.ObjectId) error {
challenges, err := model.GetChallengeListWithAchieved(nil, 0, constants.Infinite, "-_id", uid)
if err != nil {
return err
}
for _, ch := range challenges {
if err := CheckChallengeAndUpdate(ch, uid); err != nil {
continue
}
}
return nil
}

17
backend/services/challenge/create_customized_spider.go
View File

@@ -1,8 +1,23 @@
package challenge
import (
"crawlab/constants"
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type CreateCustomizedSpiderService struct {
UserId bson.ObjectId
}
func (s *CreateCustomizedSpiderService) Check() (bool, error) {
return true, nil
query := bson.M{
"user_id": s.UserId,
"type": constants.Customized,
}
_, count, err := model.GetSpiderList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return count > 0, nil
}

12
backend/services/challenge/login_7d.go
View File

@@ -1,8 +1,18 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type Login7dService struct {
UserId bson.ObjectId
}
func (s *Login7dService) Check() (bool, error) {
return true, nil
days, err := model.GetVisitDays(s.UserId)
if err != nil {
return false, err
}
return days >= 7, nil
}

17
backend/services/challenge/run_random.go
View File

@@ -1,8 +1,23 @@
package challenge
import (
"crawlab/constants"
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type RunRandomService struct {
UserId bson.ObjectId
}
func (s *RunRandomService) Check() (bool, error) {
return true, nil
query := bson.M{
"user_id": s.UserId,
"run_type": constants.RunTypeRandom,
}
list, err := model.GetTaskList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return len(list) > 0, nil
}

2
backend/services/spider.go
View File

@@ -471,6 +471,7 @@ func InitDemoSpiders () {
ProjectId: bson.ObjectIdHex(constants.ObjectIdNull),
FileId: bson.ObjectIdHex(constants.ObjectIdNull),
Cmd: configData.Cmd,
UserId: bson.ObjectIdHex(constants.ObjectIdNull),
}
if err := spider.Add(); err != nil {
log.Errorf("add spider error: " + err.Error())
@@ -497,6 +498,7 @@ func InitDemoSpiders () {
ProjectId: bson.ObjectIdHex(constants.ObjectIdNull),
FileId: bson.ObjectIdHex(constants.ObjectIdNull),
Config: configData,
UserId: bson.ObjectIdHex(constants.ObjectIdNull),
}
if err := spider.Add(); err != nil {
log.Errorf("add spider error: " + err.Error())

5
backend/services/user.go
View File

@@ -115,3 +115,8 @@ func GetCurrentUser(c *gin.Context) *model.User {
data, _ := c.Get("currentUser")
return data.(*model.User)
}
func GetCurrentUserId(c *gin.Context) bson.ObjectId {
return GetCurrentUser(c).Id
}

1
frontend/src/views/challenge/ChallengeList.vue
View File

@@ -77,6 +77,7 @@ export default {
},
methods: {
async getData () {
await this.$request.post('/challenges-check')
const res = await this.$request.get('/challenges')
this.challenges = res.data.data || []
}