From 3bd3c0117ba5e102b1ab86cc3774064490fbf939 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 19 Feb 2020 10:31:50 +0800 Subject: [PATCH 1/8] added dockerpush.yml --- .github/workflows/dockerpush.yml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/dockerpush.yml diff --git a/.github/workflows/dockerpush.yml b/.github/workflows/dockerpush.yml new file mode 100644 index 00000000..00de65ec --- /dev/null +++ b/.github/workflows/dockerpush.yml @@ -0,0 +1,53 @@ +name: Docker + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + - release + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: tikazyq/crawlab + +jobs: + # Push image to GitHub Package Registry. + # See also https://docs.docker.com/docker-hub/builds/ + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Build image + run: docker build . --file Dockerfile --tag tikazyq/crawlab + + - name: Log into registry + run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + + - name: Push image + run: | + IMAGE_ID=tikazyq/$IMAGE_NAME + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag image $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION From 2b084d867007a18be3dcfe360397ed72596e97af Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 19 Feb 2020 10:42:51 +0800 Subject: [PATCH 2/8] updated CHANGELOG --- CHANGELOG-zh.md | 7 +++++++ CHANGELOG.md | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index 38738789..7a411d38 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,3 +1,10 @@ +# 0.4.7 (unknown) +### 功能 / 优化 +- **更好的支持 Scrapy**. 爬虫识别，`settings.py` 配置，日志级别选择，爬虫选择. [#435](https://github.com/crawlab-team/crawlab/issues/435) +- **Git 同步**. 允许用户将 Git 项目同步到 Crawlab. + +### Bug 修复 + # 0.4.6 (2020-02-13) ### 功能 / 优化 - **Node.js SDK**. 用户可以将 SDK 应用到他们的 Node.js 爬虫中. diff --git a/CHANGELOG.md b/CHANGELOG.md index 707dd09d..784b2063 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# 0.4.7 (unknown) +### Features / Enhancement +- **Better Support for Scrapy**. Spiders identification, `settings.py` configuration, log level selection, spider selection. [#435](https://github.com/crawlab-team/crawlab/issues/435) +- **Git Sync**. Allow users to sync git projects to Crawlab. + +### Bug Fixes + # 0.4.6 (2020-02-13) ### Features / Enhancement - **SDK for Node.js**. Users can apply SDK in their Node.js spiders. From e4c012cf2f83a7aa6f7338c48a84152f21186f27 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 19 Feb 2020 13:16:46 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/entity/version.go | 23 +++ backend/main.go | 3 +- backend/routes/version.go | 20 +++ backend/services/version.go | 29 ++++ frontend/src/App.vue | 30 +--- .../src/views/layout/components/Navbar.vue | 142 +++++++++++++++++- 6 files changed, 218 insertions(+), 29 deletions(-) create mode 100644 backend/entity/version.go create mode 100644 backend/routes/version.go create mode 100644 backend/services/version.go diff --git a/backend/entity/version.go b/backend/entity/version.go new file mode 100644 index 00000000..97a0278d --- /dev/null +++ b/backend/entity/version.go @@ -0,0 +1,23 @@ +package entity + +type Release struct { + Name string `json:"name"` + Draft bool `json:"draft"` + PreRelease bool `json:"pre_release"` + PublishedAt string `json:"published_at"` + Body string `json:"body"` +} + +type ReleaseSlices []Release + +func (r ReleaseSlices) Len() int { + return len(r) +} + +func (r ReleaseSlices) Less(i, j int) bool { + return r[i].PublishedAt < r[j].PublishedAt +} + +func (r ReleaseSlices) Swap(i, j int) { + r[i], r[j] = r[j], r[i] +} diff --git a/backend/main.go b/backend/main.go index 7e9d1369..6c00c797 100644 --- a/backend/main.go +++ b/backend/main.go @@ -133,7 +133,8 @@ func main() { anonymousGroup.PUT("/users", routes.PutUser) // 添加用户 anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息 // release版本 - anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本 + anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本 + anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本 } authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) { diff --git a/backend/routes/version.go b/backend/routes/version.go new file mode 100644 index 00000000..ec3b80c7 --- /dev/null +++ b/backend/routes/version.go @@ -0,0 +1,20 @@ +package routes + +import ( + "crawlab/services" + "github.com/gin-gonic/gin" + "net/http" +) + +func GetLatestRelease(c *gin.Context) { + latestRelease, err := services.GetLatestRelease() + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: latestRelease, + }) +} diff --git a/backend/services/version.go b/backend/services/version.go new file mode 100644 index 00000000..34df7b22 --- /dev/null +++ b/backend/services/version.go @@ -0,0 +1,29 @@ +package services + +import ( + "crawlab/entity" + "github.com/apex/log" + "github.com/imroc/req" + "runtime/debug" + "sort" +) + +func GetLatestRelease() (release entity.Release, err error) { + res, err := req.Get("https://api.github.com/repos/crawlab-team/crawlab/releases") + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return release, err + } + + var releaseDataList entity.ReleaseSlices + if err := res.ToJSON(&releaseDataList); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return release, err + } + + sort.Sort(releaseDataList) + + return releaseDataList[len(releaseDataList)-1], nil +} diff --git a/frontend/src/App.vue b/frontend/src/App.vue index ec60d28b..d66691c4 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -35,33 +35,6 @@ export default { }, methods: {}, async mounted () { - // window.setUseStats = (value) => { - // document.querySelector('.el-message__closeBtn').click() - // if (value === 1) { - // this.$st.sendPv('/allow_stats') - // this.$st.sendEv('全局', '允许/禁止统计', '允许') - // } else { - // this.$st.sendPv('/disallow_stats') - // this.$st.sendEv('全局', '允许/禁止统计', '禁止') - // } - // localStorage.setItem('useStats', value) - // } - - // first-time user - // if (this.useStats === undefined || this.useStats === null) { - // this.$message({ - // type: 'info', - // dangerouslyUseHTMLString: true, - // showClose: true, - // duration: 0, - // message: '

' + this.$t('Do you allow us to collect some statistics to improve Crawlab?') + '

' + - // '
' + - // '' + - // '' + - // '
' - // }) - // } - // set uid if first visit if (this.uid === undefined || this.uid === null) { localStorage.setItem('uid', this.$utils.encrypt.UUID()) @@ -71,6 +44,9 @@ export default { if (this.sid === undefined || this.sid === null) { sessionStorage.setItem('sid', this.$utils.encrypt.UUID()) } + + // get latest version + await this.$store.dispatch('version/getLatestRelease') } } diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index 10c98427..72555bd7 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -1,5 +1,27 @@ - + @@ -401,7 +597,9 @@ export default { dialogVisible: false, addDialogVisible: false, crawlConfirmDialogVisible: false, + isRunningTasksDialogVisible: false, activeSpiderId: undefined, + activeSpider: undefined, filter: { project_id: '', keyword: '', @@ -539,7 +737,8 @@ export default { this.$utils.tour.nextStep('spider-list-add', currentStep) } }, - handle: undefined + handle: undefined, + activeSpiderTaskStatus: 'running' } }, computed: { @@ -576,11 +775,11 @@ export default { columns.push({ name: 'type', label: 'Spider Type', width: '120', sortable: true }) columns.push({ name: 'is_long_task', label: 'Is Long Task', width: '80' }) columns.push({ name: 'is_scrapy', label: 'Is Scrapy', width: '80' }) + columns.push({ name: 'latest_tasks', label: 'Latest Tasks', width: '180' }) columns.push({ name: 'last_status', label: 'Last Status', width: '120' }) columns.push({ name: 'last_run_ts', label: 'Last Run', width: '140' }) columns.push({ name: 'update_ts', label: 'Update Time', width: '140' }) columns.push({ name: 'create_ts', label: 'Create Time', width: '140' }) - columns.push({ name: 'running_tasks', label: 'Running Tasks', width: '120' }) columns.push({ name: 'remark', label: 'Remark', width: '140' }) return columns }, @@ -686,12 +885,6 @@ export default { onAddDialogClose () { this.addDialogVisible = false }, - onAddCustomizedDialogClose () { - this.addCustomizedDialogVisible = false - }, - onAddConfigurableDialogClose () { - this.addConfigurableDialogVisible = false - }, onEdit (row) { this.isEditMode = true this.$store.commit('spider/SET_SPIDER_FORM', row) @@ -719,6 +912,11 @@ export default { this.activeSpiderId = row._id this.$st.sendEv('爬虫列表', '点击运行') }, + onCrawlConfirm () { + setTimeout(() => { + this.getList() + }, 1000) + }, onView (row, ev) { ev.stopPropagation() this.$router.push('/spiders/' + row._id) @@ -768,21 +966,6 @@ export default { callback(data) }) }, - onAddConfigurableSiteSelect (item) { - this.spiderForm.site = item._id - }, - onAddConfigurableSpider () { - this.$refs['addConfigurableForm'].validate(res => { - if (res) { - this.addConfigurableLoading = true - this.$store.dispatch('spider/addSpider') - .finally(() => { - this.addConfigurableLoading = false - this.addConfigurableDialogVisible = false - }) - } - }) - }, onUploadSuccess (res) { // clear fileList this.fileList = [] @@ -836,14 +1019,56 @@ export default { project_id: this.filter.project_id } await this.$store.dispatch('spider/getSpiderList', params) + + // 更新当前爬虫（任务列表） + this.updateActiveSpider() }, - getRunningTasksStatusType (row) { - if (!row.running_tasks || row.running_tasks.length === 0) { - return 'info' - } else if (this.activeNodeList && row.running_tasks.length === this.activeNodeList.length) { - return 'warning' - } else { - return 'warning' + getTasksByStatus (row, status) { + if (!row.latest_tasks) return [] + return row.latest_tasks.filter(d => d.status === status) + }, + getTaskCountByStatus (row, status) { + return this.getTasksByStatus(row, status).length + }, + updateActiveSpider () { + if (this.activeSpider) { + for (let i = 0; i < this.spiderList.length; i++) { + const spider = this.spiderList[i] + if (this.activeSpider._id === spider._id) { + this.activeSpider = spider + } + } + } + }, + onViewRunningTasks (row, ev) { + ev.stopPropagation() + this.activeSpider = row + this.isRunningTasksDialogVisible = true + }, + getTasksByNode (row) { + if (!this.activeSpider.latest_tasks) { + return [] + } + return this.activeSpider.latest_tasks + .filter(d => d.node_id === row._id && d.status === this.activeSpiderTaskStatus) + .map(d => { + d = JSON.parse(JSON.stringify(d)) + d.create_ts = d.create_ts.match('^0001') ? 'NA' : dayjs(d.create_ts).format('YYYY-MM-DD HH:mm:ss') + d.start_ts = d.start_ts.match('^0001') ? 'NA' : dayjs(d.start_ts).format('YYYY-MM-DD HH:mm:ss') + d.finish_ts = d.finish_ts.match('^0001') ? 'NA' : dayjs(d.finish_ts).format('YYYY-MM-DD HH:mm:ss') + return d + }) + }, + onViewTask (row) { + this.$router.push(`/tasks/${row._id}`) + this.$st.sendEv('爬虫列表', '任务列表', '查看任务') + }, + async onStop (row, ev) { + ev.stopPropagation() + const res = await this.$store.dispatch('task/cancelTask', row._id) + if (!res.data.error) { + this.$message.success(`Task "${row._id}" has been sent signal to stop`) + this.getList() } } }, @@ -856,6 +1081,9 @@ export default { this.filter.project_id = this.$route.params.project_id } + // fetch node list + await this.$store.dispatch('node/getNodeList') + // fetch spider list await this.getList() @@ -969,4 +1197,12 @@ export default { .actions { text-align: right; } + + .el-table >>> .latest-tasks .el-tag { + margin: 3px 3px 0 0; + } + + .legend .el-tag { + margin-right: 5px; + } From c18921ce4b0edb60e8e54ab37f3803b80a956916 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 20 Feb 2020 12:28:26 +0800 Subject: [PATCH 7/8] refactor code --- .../src/components/Status/StatusLegend.vue | 40 +++++++++++++++++++ frontend/src/views/spider/SpiderList.vue | 34 ++-------------- frontend/src/views/task/TaskList.vue | 7 +++- 3 files changed, 49 insertions(+), 32 deletions(-) create mode 100644 frontend/src/components/Status/StatusLegend.vue diff --git a/frontend/src/components/Status/StatusLegend.vue b/frontend/src/components/Status/StatusLegend.vue new file mode 100644 index 00000000..7a4e443f --- /dev/null +++ b/frontend/src/components/Status/StatusLegend.vue @@ -0,0 +1,40 @@ + + + + + diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 2ddd2369..45c04f7f 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -170,7 +170,6 @@ -
- - - {{$t('Pending')}} - - - - {{$t('Running')}} - - - - {{$t('Finished')}} - - - - {{$t('Error')}} - - - - {{$t('Cancelled')}} - - - - {{$t('Abnormal')}} - -
+ @@ -578,10 +552,12 @@ import { import dayjs from 'dayjs' import CrawlConfirmDialog from '../../components/Common/CrawlConfirmDialog' import StatusTag from '../../components/Status/StatusTag' +import StatusLegend from '../../components/Status/StatusLegend' export default { name: 'SpiderList', components: { + StatusLegend, CrawlConfirmDialog, StatusTag }, @@ -1201,8 +1177,4 @@ export default { .el-table >>> .latest-tasks .el-tag { margin: 3px 3px 0 0; } - - .legend .el-tag { - margin-right: 5px; - } diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index 1eb5d484..906cd291 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -45,6 +45,10 @@ + + + + Date: Thu, 20 Feb 2020 13:51:48 +0800 Subject: [PATCH 8/8] updated CHANGELOG --- CHANGELOG-zh.md | 2 ++ CHANGELOG.md | 2 ++ frontend/src/views/spider/SpiderList.vue | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index 848e352c..e4f87aeb 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -2,6 +2,8 @@ ### 功能 / 优化 - **更好的支持 Scrapy**. 爬虫识别，`settings.py` 配置，日志级别选择，爬虫选择. [#435](https://github.com/crawlab-team/crawlab/issues/435) - **Git 同步**. 允许用户将 Git 项目同步到 Crawlab. +- **长任务支持**. 用户可以添加长任务爬虫，这些爬虫可以跑长期运行的任务. [425](https://github.com/crawlab-team/crawlab/issues/425) +- **爬虫列表优化**. 分状态任务列数统计，任务列表详情弹出框，图例. [425](https://github.com/crawlab-team/crawlab/issues/425) - **版本升级检测**. 检测最新版本，通知用户升级. ### Bug 修复 diff --git a/CHANGELOG.md b/CHANGELOG.md index cfb7f29c..5248c661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Features / Enhancement - **Better Support for Scrapy**. Spiders identification, `settings.py` configuration, log level selection, spider selection. [#435](https://github.com/crawlab-team/crawlab/issues/435) - **Git Sync**. Allow users to sync git projects to Crawlab. +- **Long Task Support**. Users can add long-task spiders which is supposed to run without finishing. [#425](https://github.com/crawlab-team/crawlab/issues/425) +- **Spider List Optimization**. Tasks count by status, tasks detail popup, legend. [#425](https://github.com/crawlab-team/crawlab/issues/425) - **Upgrade Check**. Check latest version and notifiy users to upgrade. ### Bug Fixes diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 45c04f7f..ba6751a5 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -70,6 +70,12 @@ active-color="#13ce66" /> + + +