Merge pull request #204 from wo10378931/develop

fix 几个问题

backend/model/spider.go

@@ -99,7 +99,7 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) {
 
 	// 获取爬虫列表
 	spiders := []Spider{}
-	if err := c.Find(filter).Skip(skip).Limit(limit).Sort("name asc").All(&spiders); err != nil {
+	if err := c.Find(filter).Skip(skip).Limit(limit).Sort("+name").All(&spiders); err != nil {
 		debug.PrintStack()
 		return spiders, err
 	}
@@ -177,6 +177,24 @@ func RemoveSpider(id bson.ObjectId) error {
 	return nil
 }
 
+func RemoveAllSpider() error {
+	s, c := database.GetCol("spiders")
+	defer s.Close()
+
+	spiders := []Spider{}
+	err := c.Find(nil).All(&spiders)
+	if err != nil {
+		log.Error("get all spiders error:" + err.Error())
+		return err
+	}
+	for _, spider := range spiders {
+		if err := RemoveSpider(spider.Id); err != nil {
+			log.Error("remove spider error:" + err.Error())
+		}
+	}
+	return nil
+}
+
 func GetSpiderCount() (int, error) {
 	s, c := database.GetCol("spiders")
 	defer s.Close()

backend/model/task.go

@@ -190,6 +190,21 @@ func RemoveTask(id string) error {
 	return nil
 }
 
+func RemoveTaskBySpiderId(id string) error {
+	tasks, err := GetTaskList(bson.M{"spider_id": id}, 0, constants.Infinite, "-create_ts")
+	if err != nil {
+		log.Error("get tasks error:" + err.Error())
+	}
+
+	for _, task := range tasks {
+		if err := RemoveTask(task.Id); err != nil {
+			log.Error("remove task error:" + err.Error())
+			continue
+		}
+	}
+	return nil
+}
+
 func GetTaskCount(query interface{}) (int, error) {
 	s, c := database.GetCol("tasks")
 	defer s.Close()
@@ -207,7 +222,7 @@ func GetDailyTaskStats(query bson.M) ([]TaskDailyItem, error) {
 	defer s.Close()
 
 	// 起始日期
-	startDate := time.Now().Add(- 30 * 24 * time.Hour)
+	startDate := time.Now().Add(-30 * 24 * time.Hour)
 	endDate := time.Now()
 
 	// query

backend/routes/spider.go

@@ -229,6 +229,12 @@ func DeleteSpider(c *gin.Context) {
 		return
 	}
 
+	// 删除爬虫对应的task任务
+	if err := model.RemoveTaskBySpiderId(spider.Id.Hex()); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",

backend/services/log.go

@@ -35,7 +35,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) {
 	}
 	defer f.Close()
 
-	const bufLen = 2048
+	const bufLen = 2 * 1024 * 1024
 	logBuf := make([]byte, bufLen)
 
 	off := int64(0)

backend/services/spider.go

@@ -91,16 +91,25 @@ func GetSpidersFromDir() ([]model.Spider, error) {
 
 // 将爬虫保存到数据库
 func SaveSpiders(spiders []model.Spider) error {
-	// 遍历爬虫列表
+	s, c := database.GetCol("spiders")
+	defer s.Close()
+
+	if len(spiders) == 0 {
+		err := model.RemoveAllSpider()
+		if err != nil {
+			log.Error("remove all spider error:" + err.Error())
+			return err
+		}
+		log.Info("get spider from dir is empty,removed all spider")
+		return nil
+	}
+	// 如果该爬虫不存在于数据库，则保存爬虫到数据库
 	for _, spider := range spiders {
 		// 忽略非自定义爬虫
 		if spider.Type != constants.Customized {
 			continue
 		}
 
-		// 如果该爬虫不存在于数据库，则保存爬虫到数据库
-		s, c := database.GetCol("spiders")
-		defer s.Close()
 		var spider_ *model.Spider
 		if err := c.Find(bson.M{"src": spider.Src}).One(&spider_); err != nil {
 			// 不存在
@@ -108,11 +117,8 @@ func SaveSpiders(spiders []model.Spider) error {
 				debug.PrintStack()
 				return err
 			}
-		} else {
-			// 存在
 		}
 	}
-
 	return nil
 }
 
@@ -252,8 +258,8 @@ func PublishAllSpiders() error {
 	for _, spider := range spiders {
 		// 发布爬虫
 		if err := PublishSpider(spider); err != nil {
-			log.Errorf(err.Error())
-			return err
+			log.Errorf("publish spider error:" + err.Error())
+			// return err
 		}
 	}
 
@@ -355,7 +361,7 @@ func OnFileUpload(message redis.Message) (err error) {
 	// 解压缩临时文件到目标文件夹
 	dstPath := filepath.Join(
 		viper.GetString("spider.path"),
-		//strings.Replace(msg.FileName, ".zip", "", -1),
+		// strings.Replace(msg.FileName, ".zip", "", -1),
 	)
 	if err := utils.DeCompress(tmpFile, dstPath); err != nil {
 		log.Errorf(err.Error())

frontend/src/views/spider/SpiderList.vue

@@ -275,15 +275,15 @@ export default {
       },
       // tableData,
       columns: [
-        { name: 'name', label: 'Name', width: '180', align: 'left' },
+        { name: 'name', label: 'Name', width: '160', align: 'left' },
         // { name: 'site_name', label: 'Site', width: '140', align: 'left' },
         { name: 'type', label: 'Spider Type', width: '120' },
         // { name: 'cmd', label: 'Command Line', width: '200' },
         { name: 'last_status', label: 'Last Status', width: '120' },
-        { name: 'last_run_ts', label: 'Last Run', width: '160' },
-        { name: 'create_ts', label: 'Create Time', width: '160' },
-        { name: 'update_ts', label: 'Update Time', width: '160' },
-        { name: 'remark', label: 'Remark', width: '160' }
+        { name: 'last_run_ts', label: 'Last Run', width: '140' },
+        { name: 'create_ts', label: 'Create Time', width: '140' },
+        { name: 'update_ts', label: 'Update Time', width: '140' },
+        { name: 'remark', label: 'Remark', width: '140' }
         // { name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' },
         // { name: 'last_5_errors', label: 'Last 5-Run Errors', width: '80' }
       ],