From 22b44d74fd7dbd34ae117b68139f25df779bcdaf Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sat, 8 Feb 2020 09:46:30 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E5=BC=80=E5=90=AF=E6=95=99=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/i18n/zh.js | 4 ++-- frontend/src/views/setting/Setting.vue | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index c904a8bd..9b1fc883 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -496,8 +496,8 @@ docker run -d --restart always --name crawlab_worker \\ 'The parameters which will be passed into the spider program.': '将被传入爬虫程序里的参数', 'The description for the schedule': '定时任务的描述', 'Once you have filled all fields, click this button to submit.': '当您填完所有字段，请点击这个按钮来提交定时任务', - 'Here you can set your password.': '这里您可以设置您的密码', - 'In this tab you can configure your notification settings.': '在这个标签中，您可以配置您的消息通知配置', + 'Here you can set your general settings.': '这里您可以设置您的通用设置', + 'In this tab you can configure your notification settings.': '在这个标签中，您可以
配置您的消息通知配置', 'Here you can add/edit/delete global environment variables which will be passed into your spider programs.': '这里您可以添加/修改/删除全局环境变量，它们会被传入爬虫程序中', 'You are running on a mobile device, which is not optimized yet. Please try with a laptop or desktop.': '您正在没有优化过的移动端上浏览，我们建议您用电脑来访问', diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue index 3b57fb4e..3832388d 100644 --- a/frontend/src/views/setting/Setting.vue +++ b/frontend/src/views/setting/Setting.vue @@ -300,8 +300,8 @@ export default { }, mounted () { if (!this.$utils.tour.isFinishedTour('setting')) { - // this.$tours['setting'].start() - // this.$st.sendEv('教程', '开始', 'setting') + this.$tours['setting'].start() + this.$st.sendEv('教程', '开始', 'setting') } } } From 4a62542eb1702dc97e156de1e0844735919045f6 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sat, 8 Feb 2020 09:51:11 +0800 Subject: [PATCH 2/3] fixed sinastock_spider issue --- .../spiders/sinastock/sinastock/spiders/sinastock_spider.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/backend/template/spiders/sinastock/sinastock/spiders/sinastock_spider.py b/backend/template/spiders/sinastock/sinastock/spiders/sinastock_spider.py index 54daf763..95bed149 100644 --- a/backend/template/spiders/sinastock/sinastock/spiders/sinastock_spider.py +++ b/backend/template/spiders/sinastock/sinastock/spiders/sinastock_spider.py @@ -11,12 +11,6 @@ from sinastock.items import NewsItem class SinastockSpiderSpider(scrapy.Spider): name = 'sinastock_spider' allowed_domains = ['finance.sina.com.cn'] - mongo = MongoClient( - host=os.environ.get('MONGO_HOST') or 'localhost', - port=int(os.environ.get('MONGO_PORT') or 27017) - ) - db = mongo[os.environ.get('MONGO_DB') or 'crawlab_test'] - col = db.get_collection(os.environ.get('CRAWLAB_COLLECTION') or 'stock_news') def start_requests(self): col = self.db['stocks'] From 12b03cba8a54b29c9bb84ec878f36b2cdd65c806 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sun, 9 Feb 2020 09:02:13 +0800 Subject: [PATCH 3/3] fixed https://github.com/crawlab-team/crawlab/issues/524 --- backend/model/spider.go | 4 ++++ backend/routes/spider.go | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/backend/model/spider.go b/backend/model/spider.go index 2baeb6ed..c1544259 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -78,6 +78,10 @@ func (spider *Spider) Add() error { spider.CreateTs = time.Now() spider.UpdateTs = time.Now() + if !spider.ProjectId.Valid() { + spider.ProjectId = bson.ObjectIdHex(constants.ObjectIdNull) + } + if err := c.Insert(&spider); err != nil { return err } diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 2b6dfd63..1abb6463 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -295,7 +295,12 @@ func UploadSpider(c *gin.Context) { if cmd != "" { spider.Cmd = cmd } - _ = spider.Add() + if err := spider.Add(); err != nil { + log.Error("add spider error: " + err.Error()) + debug.PrintStack() + HandleError(http.StatusInternalServerError, c, err) + return + } } else { if name != "" { spider.Name = name @@ -311,7 +316,12 @@ func UploadSpider(c *gin.Context) { } // 更新file_id spider.FileId = fid - _ = spider.Save() + if err := spider.Add(); err != nil { + log.Error("add spider error: " + err.Error()) + debug.PrintStack() + HandleError(http.StatusInternalServerError, c, err) + return + } } // 发起同步