diff --git a/CHANGELOG.md b/CHANGELOG.md index 04139677..95ef9cd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,17 @@ # 0.3.5 (2019-10-28) ### Features / Enhancement -- **Graceful Showdown**. https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385 -- **Node Info Optimization**. https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7 -- **Append System Environment Variables to Tasks**. https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3 -- **Auto Refresh Task Log**. https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3 -- **Enable HTTPS Deployment**. https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228 +- **Graceful Showdown**. [detail](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385) +- **Node Info Optimization**. [detail](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7) +- **Append System Environment Variables to Tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3) +- **Auto Refresh Task Log**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3) +- **Enable HTTPS Deployment**. [detail](https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228) ### Bug Fixes -- **Unable to fetch spider list info in schedule jobs**. https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93 -- **Unable to fetch node info from worker nodes**. https://github.com/crawlab-team/crawlab/commit/6af06efc17685a9e232e8c2b5fd819ec7d2d1674 -- **Unable to select node when trying to run spider tasks**. https://github.com/crawlab-team/crawlab/commit/31f8e03234426e97aed9b0bce6a50562f957edad +- **Unable to fetch spider list info in schedule jobs**. [detail](https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93) +- **Unable to fetch node info from worker nodes**. [detail](https://github.com/crawlab-team/crawlab/commit/6af06efc17685a9e232e8c2b5fd819ec7d2d1674) +- **Unable to select node when trying to run spider tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/31f8e03234426e97aed9b0bce6a50562f957edad) +- **Unable to fetch result count when result volume is large**. [#260](https://github.com/crawlab-team/crawlab/issues/260) +- **Node issue in schedule tasks**. [#244](https://github.com/crawlab-team/crawlab/issues/244) # 0.3.1 (2019-08-25) diff --git a/backend/model/log.go b/backend/model/log.go index abb77ed9..77e5094f 100644 --- a/backend/model/log.go +++ b/backend/model/log.go @@ -24,7 +24,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) { } defer utils.Close(f) - const bufLen = 1 * 1024 * 1024 + const bufLen = 2 * 1024 * 1024 logBuf := make([]byte, bufLen) off := int64(0) diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 36799ac3..ef758fb6 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -5,6 +5,7 @@ import ( "crawlab/database" "crawlab/lib/cron" "github.com/apex/log" + "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "runtime/debug" "time" @@ -93,7 +94,7 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) { // 获取爬虫名称 spider, err := GetSpider(schedule.SpiderId) - if err != nil { + if err != nil && err == mgo.ErrNotFound { log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error()) debug.PrintStack() _ = schedule.Delete() diff --git a/backend/model/task.go b/backend/model/task.go index df046ecc..64f06cd7 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -117,7 +117,7 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas for i, task := range tasks { // 获取爬虫名称 spider, err := task.GetSpider() - if spider.Id.Hex() == "" || err != nil { + if err != nil || spider.Id.Hex() == "" { _ = spider.Delete() } else { tasks[i].SpiderName = spider.DisplayName diff --git a/backend/utils/helpers.go b/backend/utils/helpers.go index 541d9002..1c8c96e3 100644 --- a/backend/utils/helpers.go +++ b/backend/utils/helpers.go @@ -1,6 +1,8 @@ package utils import ( + "context" + "crawlab/database" "crawlab/entity" "encoding/json" "github.com/apex/log" @@ -40,3 +42,22 @@ func Close(c io.Closer) { log.WithError(err).Error("关闭资源文件失败。") } } + +func Pub(channel string, msg entity.NodeMessage) error { + if _, err := database.RedisClient.Publish(channel, GetJson(msg)); err != nil { + log.Errorf("publish redis error: %s", err.Error()) + debug.PrintStack() + return err + } + return nil +} + +func Sub(channel string, consume database.ConsumeFunc) error { + ctx := context.Background() + if err := database.RedisClient.Subscribe(ctx, consume, channel); err != nil { + log.Errorf("subscribe redis error: %s", err.Error()) + debug.PrintStack() + return err + } + return nil +} diff --git a/frontend/package.json b/frontend/package.json index 60ac5cc8..5b835a60 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "crawlab", - "version": "0.3.4", + "version": "0.3.5", "private": true, "scripts": { "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development", diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index e294ad0c..0bd4c6d1 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -9,7 +9,7 @@ - v0.3.4 + v0.3.5 {{$t('Logout')}}