Merge pull request #282 from crawlab-team/develop

Develop

CHANGELOG.md

@@ -1,15 +1,17 @@
 # 0.3.5 (2019-10-28)
 ### Features / Enhancement
-- **Graceful Showdown**. https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385
-- **Node Info Optimization**. https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7
-- **Append System Environment Variables to Tasks**. https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3
-- **Auto Refresh Task Log**. https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3
-- **Enable HTTPS Deployment**. https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228
+- **Graceful Showdown**. [detail](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)
+- **Node Info Optimization**. [detail](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7)
+- **Append System Environment Variables to Tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)
+- **Auto Refresh Task Log**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)
+- **Enable HTTPS Deployment**. [detail](https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228)
 
 ### Bug Fixes
-- **Unable to fetch spider list info in schedule jobs**. https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93
-- **Unable to fetch node info from worker nodes**. https://github.com/crawlab-team/crawlab/commit/6af06efc17685a9e232e8c2b5fd819ec7d2d1674
-- **Unable to select node when trying to run spider tasks**. https://github.com/crawlab-team/crawlab/commit/31f8e03234426e97aed9b0bce6a50562f957edad
+- **Unable to fetch spider list info in schedule jobs**. [detail](https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93)
+- **Unable to fetch node info from worker nodes**. [detail](https://github.com/crawlab-team/crawlab/commit/6af06efc17685a9e232e8c2b5fd819ec7d2d1674)
+- **Unable to select node when trying to run spider tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/31f8e03234426e97aed9b0bce6a50562f957edad)
+- **Unable to fetch result count when result volume is large**. [#260](https://github.com/crawlab-team/crawlab/issues/260)
+- **Node issue in schedule tasks**. [#244](https://github.com/crawlab-team/crawlab/issues/244)
 
 
 # 0.3.1 (2019-08-25)

backend/model/log.go

@@ -24,7 +24,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) {
 	}
 	defer utils.Close(f)
 
-	const bufLen = 1 * 1024 * 1024
+	const bufLen = 2 * 1024 * 1024
 	logBuf := make([]byte, bufLen)
 
 	off := int64(0)

backend/model/schedule.go

@@ -5,6 +5,7 @@ import (
 	"crawlab/database"
 	"crawlab/lib/cron"
 	"github.com/apex/log"
+	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"runtime/debug"
 	"time"
@@ -93,7 +94,7 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 
 		// 获取爬虫名称
 		spider, err := GetSpider(schedule.SpiderId)
-		if err != nil {
+		if err != nil && err == mgo.ErrNotFound {
 			log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error())
 			debug.PrintStack()
 			_ = schedule.Delete()

backend/model/task.go

@@ -117,7 +117,7 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas
 	for i, task := range tasks {
 		// 获取爬虫名称
 		spider, err := task.GetSpider()
-		if spider.Id.Hex() == "" || err != nil {
+		if err != nil || spider.Id.Hex() == "" {
 			_ = spider.Delete()
 		} else {
 			tasks[i].SpiderName = spider.DisplayName

backend/utils/helpers.go

@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"context"
+	"crawlab/database"
 	"crawlab/entity"
 	"encoding/json"
 	"github.com/apex/log"
@@ -40,3 +42,22 @@ func Close(c io.Closer) {
 		log.WithError(err).Error("关闭资源文件失败。")
 	}
 }
+
+func Pub(channel string, msg entity.NodeMessage) error {
+	if _, err := database.RedisClient.Publish(channel, GetJson(msg)); err != nil {
+		log.Errorf("publish redis error: %s", err.Error())
+		debug.PrintStack()
+		return err
+	}
+	return nil
+}
+
+func Sub(channel string, consume database.ConsumeFunc) error {
+	ctx := context.Background()
+	if err := database.RedisClient.Subscribe(ctx, consume, channel); err != nil {
+		log.Errorf("subscribe redis error: %s", err.Error())
+		debug.PrintStack()
+		return err
+	}
+	return nil
+}

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlab",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "scripts": {
     "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development",

frontend/src/views/layout/components/Navbar.vue

@@ -9,7 +9,7 @@
       </span>
       <el-dropdown-menu slot="dropdown" class="user-dropdown">
         <el-dropdown-item>
-          <span style="display:block;">v0.3.4</span>
+          <span style="display:block;">v0.3.5</span>
         </el-dropdown-item>
         <el-dropdown-item>
           <span style="display:block;" @click="logout">{{$t('Logout')}}</span>