diff --git a/backend/constants/auth.go b/backend/constants/auth.go
new file mode 100644
index 00000000..136391a0
--- /dev/null
+++ b/backend/constants/auth.go
@@ -0,0 +1,7 @@
+package constants
+
+const (
+ OwnerTypeAll = "all"
+ OwnerTypeMe = "me"
+ OwnerTypePublic = "public"
+)
diff --git a/backend/main.go b/backend/main.go
index ac76a453..110bdecc 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -100,6 +100,14 @@ func main() {
panic(err)
}
log.Info("initialized challenge service successfully")
+
+ // 初始化清理服务
+ if err := services.InitCleanService(); err != nil {
+ log.Error("init clean service error:" + err.Error())
+ debug.PrintStack()
+ panic(err)
+ }
+ log.Info("initialized clean service successfully")
}
// 初始化任务执行器
diff --git a/backend/model/project.go b/backend/model/project.go
index bc7ca35e..09f52f41 100644
--- a/backend/model/project.go
+++ b/backend/model/project.go
@@ -16,11 +16,12 @@ type Project struct {
Tags []string `json:"tags" bson:"tags"`
// 前端展示
- Spiders []Spider `json:"spiders" bson:"spiders"`
+ Spiders []Spider `json:"spiders" bson:"spiders"`
+ Username string `json:"username" bson:"username"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
- CreateTs time.Time `json:"create_ts" bson:"create_ts"`
- UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
+ CreateTs time.Time `json:"create_ts" bson:"create_ts"`
+ UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
}
func (p *Project) Save() error {
@@ -90,15 +91,21 @@ func GetProject(id bson.ObjectId) (Project, error) {
return p, nil
}
-func GetProjectList(filter interface{}, skip int, sortKey string) ([]Project, error) {
+func GetProjectList(filter interface{}, sortKey string) ([]Project, error) {
s, c := database.GetCol("projects")
defer s.Close()
var projects []Project
- if err := c.Find(filter).Skip(skip).Limit(constants.Infinite).Sort(sortKey).All(&projects); err != nil {
+ if err := c.Find(filter).Sort(sortKey).All(&projects); err != nil {
debug.PrintStack()
return projects, err
}
+
+ for i, p := range projects {
+ // 获取用户名称
+ user, _ := GetUser(p.UserId)
+ projects[i].Username = user.Username
+ }
return projects, nil
}
diff --git a/backend/model/schedule.go b/backend/model/schedule.go
index a23b6973..ee4028af 100644
--- a/backend/model/schedule.go
+++ b/backend/model/schedule.go
@@ -29,6 +29,7 @@ type Schedule struct {
// 前端展示
SpiderName string `json:"spider_name" bson:"spider_name"`
+ Username string `json:"user_name" bson:"user_name"`
Nodes []Node `json:"nodes" bson:"nodes"`
Message string `json:"message" bson:"message"`
@@ -83,6 +84,10 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
schedule.SpiderName = spider.Name
}
+ // 获取用户名称
+ user, _ := GetUser(schedule.UserId)
+ schedule.Username = user.Username
+
schs = append(schs, schedule)
}
return schs, nil
@@ -92,11 +97,16 @@ func GetSchedule(id bson.ObjectId) (Schedule, error) {
s, c := database.GetCol("schedules")
defer s.Close()
- var result Schedule
- if err := c.FindId(id).One(&result); err != nil {
- return result, err
+ var schedule Schedule
+ if err := c.FindId(id).One(&schedule); err != nil {
+ return schedule, err
}
- return result, nil
+
+ // 获取用户名称
+ user, _ := GetUser(schedule.UserId)
+ schedule.Username = user.Username
+
+ return schedule, nil
}
func UpdateSchedule(id bson.ObjectId, item Schedule) error {
@@ -147,11 +157,11 @@ func RemoveSchedule(id bson.ObjectId) error {
return nil
}
-func GetScheduleCount() (int, error) {
+func GetScheduleCount(filter interface{}) (int, error) {
s, c := database.GetCol("schedules")
defer s.Close()
- count, err := c.Count()
+ count, err := c.Find(filter).Count()
if err != nil {
return 0, err
}
diff --git a/backend/model/spider.go b/backend/model/spider.go
index 5bc2579b..9a709b41 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -33,6 +33,7 @@ type Spider struct {
Remark string `json:"remark" bson:"remark"` // 备注
Src string `json:"src" bson:"src"` // 源码位置
ProjectId bson.ObjectId `json:"project_id" bson:"project_id"` // 项目ID
+ IsPublic bool `json:"is_public" bson:"is_public"` // 是否公开
// 自定义爬虫
Cmd string `json:"cmd" bson:"cmd"` // 执行命令
@@ -63,6 +64,7 @@ type Spider struct {
LastStatus string `json:"last_status"` // 最后执行状态
Config entity.ConfigSpiderData `json:"config"` // 可配置爬虫配置
LatestTasks []Task `json:"latest_tasks"` // 最近任务列表
+ Username string `json:"username""`
// 时间
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
@@ -83,6 +85,7 @@ func (spider *Spider) Save() error {
}
if err := c.UpdateId(spider.Id, spider); err != nil {
+ log.Errorf(err.Error())
debug.PrintStack()
return err
}
@@ -182,10 +185,22 @@ func GetSpiderList(filter interface{}, skip int, limit int, sortStr string) ([]S
continue
}
+ // 获取用户
+ var user User
+ if spider.UserId.Valid() {
+ user, err = GetUser(spider.UserId)
+ if err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ continue
+ }
+ }
+
// 赋值
spiders[i].LastRunTs = task.CreateTs
spiders[i].LastStatus = task.Status
spiders[i].LatestTasks = latestTasks
+ spiders[i].Username = user.Username
}
count, _ := c.Find(filter).Count()
@@ -221,13 +236,21 @@ func GetSpiderByName(name string) Spider {
s, c := database.GetCol("spiders")
defer s.Close()
- var result Spider
- if err := c.Find(bson.M{"name": name}).One(&result); err != nil && err != mgo.ErrNotFound {
+ var spider Spider
+ if err := c.Find(bson.M{"name": name}).One(&spider); err != nil && err != mgo.ErrNotFound {
log.Errorf("get spider error: %s, spider_name: %s", err.Error(), name)
//debug.PrintStack()
- return result
+ return spider
}
- return result
+
+ // 获取用户
+ var user User
+ if spider.UserId.Valid() {
+ user, _ = GetUser(spider.UserId)
+ }
+ spider.Username = user.Username
+
+ return spider
}
// 获取爬虫(根据ID)
@@ -253,6 +276,14 @@ func GetSpider(id bson.ObjectId) (Spider, error) {
}
spider.Config = config
}
+
+ // 获取用户名称
+ var user User
+ if spider.UserId.Valid() {
+ user, _ = GetUser(spider.UserId)
+ }
+ spider.Username = user.Username
+
return spider, nil
}
@@ -324,11 +355,11 @@ func RemoveAllSpider() error {
}
// 获取爬虫总数
-func GetSpiderCount() (int, error) {
+func GetSpiderCount(filter interface{}) (int, error) {
s, c := database.GetCol("spiders")
defer s.Close()
- count, err := c.Count()
+ count, err := c.Find(filter).Count()
if err != nil {
return 0, err
}
diff --git a/backend/model/task.go b/backend/model/task.go
index 6b2a44c1..75edd631 100644
--- a/backend/model/task.go
+++ b/backend/model/task.go
@@ -31,6 +31,7 @@ type Task struct {
// 前端数据
SpiderName string `json:"spider_name"`
NodeName string `json:"node_name"`
+ Username string `json:"username"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
@@ -128,6 +129,10 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas
if node, err := task.GetNode(); err == nil {
tasks[i].NodeName = node.Name
}
+
+ // 获取用户名称
+ user, _ := GetUser(task.UserId)
+ task.Username = user.Username
}
return tasks, nil
}
@@ -156,6 +161,11 @@ func GetTask(id string) (Task, error) {
debug.PrintStack()
return task, err
}
+
+ // 获取用户名称
+ user, _ := GetUser(task.UserId)
+ task.Username = user.Username
+
return task, nil
}
diff --git a/backend/routes/projects.go b/backend/routes/project.go
similarity index 95%
rename from backend/routes/projects.go
rename to backend/routes/project.go
index 34b2d7f4..f0dd1198 100644
--- a/backend/routes/projects.go
+++ b/backend/routes/project.go
@@ -4,6 +4,7 @@ import (
"crawlab/constants"
"crawlab/database"
"crawlab/model"
+ "crawlab/services"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
@@ -18,8 +19,11 @@ func GetProjectList(c *gin.Context) {
query["tags"] = tag
}
+ // 获取校验
+ query = services.GetAuthQuery(query, c)
+
// 获取列表
- projects, err := model.GetProjectList(query, 0, "+_id")
+ projects, err := model.GetProjectList(query, "+_id")
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
@@ -74,6 +78,9 @@ func PutProject(c *gin.Context) {
return
}
+ // UserId
+ p.UserId = services.GetCurrentUserId(c)
+
if err := p.Add(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go
index 601d467f..27ad7825 100644
--- a/backend/routes/schedule.go
+++ b/backend/routes/schedule.go
@@ -9,7 +9,12 @@ import (
)
func GetScheduleList(c *gin.Context) {
- results, err := model.GetScheduleList(nil)
+ query := bson.M{}
+
+ // 获取校验
+ query = services.GetAuthQuery(query, c)
+
+ results, err := model.GetScheduleList(query)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
diff --git a/backend/routes/spider.go b/backend/routes/spider.go
index 049e69b2..6f4d88b3 100644
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -29,13 +29,14 @@ import (
// ======== 爬虫管理 ========
func GetSpiderList(c *gin.Context) {
- pageNum, _ := c.GetQuery("page_num")
- pageSize, _ := c.GetQuery("page_size")
- keyword, _ := c.GetQuery("keyword")
- pid, _ := c.GetQuery("project_id")
- t, _ := c.GetQuery("type")
- sortKey, _ := c.GetQuery("sort_key")
- sortDirection, _ := c.GetQuery("sort_direction")
+ pageNum := c.Query("page_num")
+ pageSize := c.Query("page_size")
+ keyword := c.Query("keyword")
+ pid := c.Query("project_id")
+ t := c.Query("type")
+ sortKey := c.Query("sort_key")
+ sortDirection := c.Query("sort_direction")
+ ownerType := c.Query("owner_type")
// 筛选-名称
filter := bson.M{
@@ -65,6 +66,21 @@ func GetSpiderList(c *gin.Context) {
filter["project_id"] = bson.ObjectIdHex(pid)
}
+ // 筛选-用户
+ if ownerType == constants.OwnerTypeAll {
+ user := services.GetCurrentUser(c)
+ if user.Role == constants.RoleNormal {
+ filter["$or"] = []bson.M{
+ {"user_id": services.GetCurrentUserId(c)},
+ {"is_public": true},
+ }
+ }
+ } else if ownerType == constants.OwnerTypeMe {
+ filter["user_id"] = services.GetCurrentUserId(c)
+ } else if ownerType == constants.OwnerTypePublic {
+ filter["is_public"] = true
+ }
+
// 排序
sortStr := "-_id"
if sortKey != "" && sortDirection != "" {
@@ -815,7 +831,7 @@ func GetSpiderStats(c *gin.Context) {
overview.AvgWaitDuration = overview.TotalWaitDuration / taskCount
overview.AvgRuntimeDuration = overview.TotalRuntimeDuration / taskCount
- items, err := model.GetDailyTaskStats(bson.M{"spider_id": spider.Id})
+ items, err := model.GetDailyTaskStats(bson.M{"spider_id": spider.Id, "user_id": bson.M{"user_id": services.GetCurrentUserId(c)}})
if err != nil {
log.Errorf(err.Error())
HandleError(http.StatusInternalServerError, c, err)
diff --git a/backend/routes/stats.go b/backend/routes/stats.go
index 8590bbd7..497083e5 100644
--- a/backend/routes/stats.go
+++ b/backend/routes/stats.go
@@ -3,6 +3,7 @@ package routes
import (
"crawlab/constants"
"crawlab/model"
+ "crawlab/services"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
@@ -22,7 +23,7 @@ func GetHomeStats(c *gin.Context) {
}
// 任务总数
- taskCount, err := model.GetTaskCount(nil)
+ taskCount, err := model.GetTaskCount(bson.M{"user_id": services.GetCurrentUserId(c)})
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
@@ -36,21 +37,21 @@ func GetHomeStats(c *gin.Context) {
}
// 爬虫总数
- spiderCount, err := model.GetSpiderCount()
+ spiderCount, err := model.GetSpiderCount(bson.M{"user_id": services.GetCurrentUserId(c)})
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
// 定时任务数
- scheduleCount, err := model.GetScheduleCount()
+ scheduleCount, err := model.GetScheduleCount(bson.M{"user_id": services.GetCurrentUserId(c)})
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
// 每日任务数
- items, err := model.GetDailyTaskStats(bson.M{})
+ items, err := model.GetDailyTaskStats(bson.M{"user_id": services.GetCurrentUserId(c)})
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
diff --git a/backend/routes/task.go b/backend/routes/task.go
index 7e772ca5..2ab1a046 100644
--- a/backend/routes/task.go
+++ b/backend/routes/task.go
@@ -47,11 +47,14 @@ func GetTaskList(c *gin.Context) {
if data.SpiderId != "" {
query["spider_id"] = bson.ObjectIdHex(data.SpiderId)
}
- //新增根据任务状态获取task列表
+ // 根据任务状态获取task列表
if data.Status != "" {
query["status"] = data.Status
}
+ // 获取校验
+ query = services.GetAuthQuery(query, c)
+
// 获取任务列表
tasks, err := model.GetTaskList(query, (data.PageNum-1)*data.PageSize, data.PageSize, "-create_ts")
if err != nil {
diff --git a/backend/services/auth.go b/backend/services/auth.go
new file mode 100644
index 00000000..096d9f14
--- /dev/null
+++ b/backend/services/auth.go
@@ -0,0 +1,20 @@
+package services
+
+import (
+ "crawlab/constants"
+ "github.com/gin-gonic/gin"
+ "github.com/globalsign/mgo/bson"
+)
+
+func GetAuthQuery(query bson.M, c *gin.Context) bson.M {
+ user := GetCurrentUser(c)
+ if user.Role == constants.RoleAdmin {
+ // 获得所有数据
+ return query
+ } else {
+ // 只获取自己的数据
+ query["user_id"] = user.Id
+ return query
+ }
+}
+
diff --git a/backend/services/clean.go b/backend/services/clean.go
new file mode 100644
index 00000000..bbd3571d
--- /dev/null
+++ b/backend/services/clean.go
@@ -0,0 +1,122 @@
+package services
+
+import (
+ "crawlab/constants"
+ "crawlab/model"
+ "github.com/apex/log"
+ "github.com/globalsign/mgo/bson"
+ "runtime/debug"
+)
+
+func InitTaskCleanUserIds() {
+ adminUser, err := GetAdminUser()
+ if err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ return
+ }
+ tasks, err := model.GetTaskList(nil, 0, constants.Infinite, "+_id")
+ if err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ return
+ }
+ for _, t := range tasks {
+ if !t.ScheduleId.Valid() {
+ t.ScheduleId = bson.ObjectIdHex(constants.ObjectIdNull)
+ if err := t.Save(); err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ continue
+ }
+ }
+
+ if !t.UserId.Valid() {
+ t.UserId = adminUser.Id
+ if err := t.Save(); err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ continue
+ }
+ }
+ }
+}
+
+func InitProjectCleanUserIds() {
+ adminUser, err := GetAdminUser()
+ if err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ return
+ }
+ projects, err := model.GetProjectList(nil, "+_id")
+ if err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ return
+ }
+ for _, p := range projects {
+ if !p.UserId.Valid() {
+ p.UserId = adminUser.Id
+ if err := p.Save(); err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ continue
+ }
+ }
+ }
+}
+
+func InitSpiderCleanUserIds() {
+ adminUser, err := GetAdminUser()
+ if err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ return
+ }
+ spiders, _ := model.GetSpiderAllList(nil)
+ for _, s := range spiders {
+ if !s.UserId.Valid() {
+ s.UserId = adminUser.Id
+ if err := s.Save(); err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ continue
+ }
+ }
+ }
+}
+
+func InitScheduleCleanUserIds() {
+ adminUser, err := GetAdminUser()
+ if err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ return
+ }
+ schedules, _ := model.GetScheduleList(nil)
+ for _, s := range schedules {
+ if !s.UserId.Valid() {
+ s.UserId = adminUser.Id
+ if err := s.Save(); err != nil {
+ log.Errorf(err.Error())
+ debug.PrintStack()
+ continue
+ }
+ }
+ }
+}
+
+func InitCleanService() error {
+ if model.IsMaster() {
+ // 清理任务UserIds
+ InitTaskCleanUserIds()
+ // 清理项目UserIds
+ InitProjectCleanUserIds()
+ // 清理爬虫UserIds
+ InitSpiderCleanUserIds()
+ // 清理定时任务UserIds
+ InitScheduleCleanUserIds()
+ }
+ return nil
+}
diff --git a/backend/services/spider.go b/backend/services/spider.go
index 0fbc5ebb..f9623316 100644
--- a/backend/services/spider.go
+++ b/backend/services/spider.go
@@ -545,6 +545,9 @@ func InitSpiderService() error {
if err := GitCron.Start(); err != nil {
return err
}
+
+ // 清理UserId
+ InitSpiderCleanUserIds()
}
return nil
diff --git a/backend/services/user.go b/backend/services/user.go
index 5e283044..adc56136 100644
--- a/backend/services/user.go
+++ b/backend/services/user.go
@@ -120,3 +120,11 @@ func GetCurrentUser(c *gin.Context) *model.User {
func GetCurrentUserId(c *gin.Context) bson.ObjectId {
return GetCurrentUser(c).Id
}
+
+func GetAdminUser() (user *model.User, err error) {
+ u, err := model.GetUserByUsername("admin")
+ if err != nil {
+ return user, err
+ }
+ return &u, nil
+}
diff --git a/frontend/src/components/File/FileDetail.vue b/frontend/src/components/File/FileDetail.vue
index ee88cef7..92143019 100644
--- a/frontend/src/components/File/FileDetail.vue
+++ b/frontend/src/components/File/FileDetail.vue
@@ -1,12 +1,18 @@
-
+
+
+