From 177dcb5aba7b7f8c29fa39d3f0f0245d25cc035a Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 30 Jan 2020 14:47:50 +0800 Subject: [PATCH 01/10] updated jenkins/master/docker-compose.yaml --- jenkins/master/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/master/docker-compose.yaml b/jenkins/master/docker-compose.yaml index ae16079c..f23be242 100644 --- a/jenkins/master/docker-compose.yaml +++ b/jenkins/master/docker-compose.yaml @@ -41,6 +41,5 @@ services: - "/opt/crawlab/redis/data:/data" splash: # use Splash to run spiders on dynamic pages image: scrapinghub/splash - container_name: splash # ports: # - "8050:8050" From 1385288b73e5ceb67a0e1a6a953ac33a43210284 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 30 Jan 2020 17:36:38 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=AF=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E6=97=A0=E6=B3=95=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=20"//"=20=E6=89=93=E5=A4=B4=E7=9A=84URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/template/scrapy/config_spider/spiders/spider.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/template/scrapy/config_spider/spiders/spider.py b/backend/template/scrapy/config_spider/spiders/spider.py index 0e3c661d..d87f4297 100644 --- a/backend/template/scrapy/config_spider/spiders/spider.py +++ b/backend/template/scrapy/config_spider/spiders/spider.py @@ -2,11 +2,14 @@ import scrapy import re from config_spider.items import Item -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse def get_real_url(response, url): - if re.search(r'^https?|^\/\/', url): + if re.search(r'^https?', url): return url + elif re.search(r'^\/\/', url): + u = urlparse(response.url) + return u.scheme + url return urljoin(response.url, url) class ConfigSpider(scrapy.Spider): From 46c9ba74bd9f885ddb290481834e3a105f95687e Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 31 Jan 2020 13:00:25 +0800 Subject: [PATCH 03/10] updated README --- CHANGELOG-zh.md | 11 +++++++++++ CHANGELOG.md | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index d0f85f2a..c8be9eef 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,3 +1,14 @@ +# 0.4.5 (unkown) +### 功能 / 优化 +- **交互式教程**. 引导用户了解 Crawlab 的主要功能. + +### Bug 修复 +- **点击删除按钮导致跳转**. [#480](https://github.com/crawlab-team/crawlab/issues/480) +- **无法在空爬虫里创建文件**. [#479](https://github.com/crawlab-team/crawlab/issues/479) +- **下载结果错误**. [#465](https://github.com/crawlab-team/crawlab/issues/465) +- **crawlab-sdk CLI 错误**. [#458](https://github.com/crawlab-team/crawlab/issues/458) +- **修复“删除爬虫后获取所有爬虫”错误**. + # 0.4.4 (2020-01-17) ### 功能 / 优化 diff --git a/CHANGELOG.md b/CHANGELOG.md index c7d5f6e5..072ba9aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# 0.4.5 (unkown) +### Features / Enhancement +- **Interactive Tutorial**. Guide users through the main functionalities of Crawlab. + +### Bug Fixes +- **Click delete button results in redirect**. [#480](https://github.com/crawlab-team/crawlab/issues/480) +- **Unable to create files in an empty spider**. [#479](https://github.com/crawlab-team/crawlab/issues/479) +- **Download results error**. [#465](https://github.com/crawlab-team/crawlab/issues/465) +- **crawlab-sdk CLI error**. [#458](https://github.com/crawlab-team/crawlab/issues/458) +- **Fix getting all spider after deleting a spider**. + # 0.4.4 (2020-01-17) ### Features / Enhancement - **Email Notification**. Allow users to send email notifications. From 872916cc4a15b35e198bcc213d14f2c6b91da1bc Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 31 Jan 2020 13:55:02 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=8F=AF=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E6=95=99=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/Config/ConfigList.vue | 117 +++++++++++++++++- frontend/src/views/spider/SpiderDetail.vue | 6 + 2 files changed, 120 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index b1a10941..ed5b12d5 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -1,5 +1,14 @@