From 1339877b7422a8c4bc2ac44ba4dd421e989d3d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 4 Nov 2019 19:58:26 +0800 Subject: [PATCH 001/375] =?UTF-8?q?=E5=85=B3=E9=97=AD=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 13 +++++++++++-- backend/utils/helpers.go | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/services/task.go b/backend/services/task.go index 9e584e82..67c3396b 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -136,8 +136,15 @@ func FinishOrCancelTask(ch chan string, cmd *exec.Cmd, t model.Task) { log.Infof("process received signal: %s", signal) if signal == constants.TaskCancel && cmd.Process != nil { + var err error + // 兼容windows + if runtime.GOOS == constants.Windows { + err = cmd.Process.Kill() + } else { + err = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } // 取消进程 - if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { + if err != nil { log.Errorf("process kill error: %s", err.Error()) debug.PrintStack() @@ -225,7 +232,9 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e go FinishOrCancelTask(ch, cmd, t) // kill的时候，可以kill所有的子进程 - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if runtime.GOOS != constants.Windows { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + } // 启动进程 if err := StartTaskProcess(cmd, t); err != nil { diff --git a/backend/utils/helpers.go b/backend/utils/helpers.go index 541d9002..8a80e9e8 100644 --- a/backend/utils/helpers.go +++ b/backend/utils/helpers.go @@ -37,6 +37,6 @@ func GetMessage(message redis.Message) *entity.NodeMessage { func Close(c io.Closer) { err := c.Close() if err != nil { - log.WithError(err).Error("关闭资源文件失败。") + //log.WithError(err).Error("关闭资源文件失败。") } } From a237a0f144eb9da8986e42cccc85aae9720c21ca Mon Sep 17 00:00:00 2001 From: van Date: Wed, 6 Nov 2019 12:25:29 +0800 Subject: [PATCH 002/375] add ansi-to-html dependence, show task log ansi with colors --- frontend/package.json | 1 + frontend/src/components/ScrollView/LogItem.vue | 7 ++++++- frontend/src/components/ScrollView/LogView.vue | 8 ++++++-- frontend/yarn.lock | 9 ++++++++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 5b835a60..5f19fd7b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,6 +18,7 @@ "@fortawesome/free-solid-svg-icons": "^5.9.0", "@fortawesome/vue-fontawesome": "^0.1.6", "@tinymce/tinymce-vue": "^2.0.0", + "ansi-to-html": "^0.6.13", "axios": "0.18.0", "cross-env": "^5.2.0", "dayjs": "^1.8.6", diff --git a/frontend/src/components/ScrollView/LogItem.vue b/frontend/src/components/ScrollView/LogItem.vue index 5850086f..c54d7c37 100644 --- a/frontend/src/components/ScrollView/LogItem.vue +++ b/frontend/src/components/ScrollView/LogItem.vue @@ -1,7 +1,7 @@ @@ -16,11 +16,16 @@ export default { data: { type: String, default: '' + }, + isAnsi: { + type: Boolean, + default: false } } } + diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue index 836a8f49..7b1ef9f5 100644 --- a/frontend/src/components/TableView/FieldsTableView.vue +++ b/frontend/src/components/TableView/FieldsTableView.vue @@ -1,73 +1,104 @@ @@ -250,7 +250,8 @@ export default { onSelectSpider () { this.$st.sendEv('任务', '选择爬虫') }, - onRemove (row) { + onRemove (row, ev) { + ev.stopPropagation() this.$confirm(this.$t('Are you sure to delete this task?'), this.$t('Notification'), { confirmButtonText: this.$t('Confirm'), cancelButtonText: this.$t('Cancel'), From 550d98326ef1445d74f8b8000e3e0cdaeb0e04ac Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 28 Nov 2019 12:54:56 +0800 Subject: [PATCH 038/375] =?UTF-8?q?fixed=20=E9=85=8D=E7=BD=AE=E4=B8=8B?= =?UTF-8?q?=E5=88=92=E7=BA=BF=E4=B8=8D=E6=98=BE=E7=A4=BA=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/Config/ConfigList.vue | 10 ++++++++++ frontend/src/views/spider/SpiderDetail.vue | 8 ++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 7b7c7ffd..1c127ca3 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -538,6 +538,16 @@ export default { }, onTabClick (tab) { this.activeTab = tab.name + }, + update () { + if (this.activeTab !== 'stages') return + + // 手动显示tab下划线 + const elBar = document.querySelector('.el-tabs__active-bar') + const elStages = document.querySelector('#tab-stages') + const totalWidth = Number(getComputedStyle(elStages).width.replace('px', '')) + const paddingRight = Number(getComputedStyle(elStages).paddingRight.replace('px', '')) + elBar.setAttribute('style', 'width:' + (totalWidth - paddingRight) + 'px') } }, created () { diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index b42e750d..29b531ef 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -13,8 +13,8 @@ - - + + @@ -77,6 +77,10 @@ export default { setTimeout(() => { this.$refs['spider-stats'].update() }, 0) + } else if (this.activeTabName === 'config') { + setTimeout(() => { + this.$refs['config'].update() + }, 0) } this.$st.sendEv('爬虫详情', '切换标签', tab.name) }, From 6e26b7fb42c2d7ed3e476daeafc730d5856bf050 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 28 Nov 2019 13:32:33 +0800 Subject: [PATCH 039/375] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8F=AF=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E7=88=AC=E8=99=AB=E5=89=8D=E7=AB=AF=E7=95=8C=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/Config/ConfigList.vue | 27 +++++++++++++++++-- .../components/TableView/FieldsTableView.vue | 15 ++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 1c127ca3..e7c7212a 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -493,6 +493,7 @@ export default { }, series: [ { + animation: false, type: 'graph', layout: 'force', symbolSize: 50, @@ -515,7 +516,7 @@ export default { force: { initLayout: 'force', repulsion: 100, - gravity: 0.01, + gravity: 0.00001, edgeLength: 200 }, // draggable: true, @@ -529,7 +530,29 @@ export default { } } } - ] + ], + tooltip: { + // formatter: '{b0}: {c0}
{b1}: {c1}', + formatter: (params) => { + if (!params.data.fields) return + + let str = '' + str += `

` + str += `
` + str += '
    ' + for (let i = 0; i < params.data.fields.length; i++) { + const f = params.data.fields[i] + str += ` +
  • +${f.name}: +${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' --> ' + '' + f.next_stage + '') : ''} +
    • +` + } + str += '
' + return str + } + } } const el = document.querySelector('#process-chart') this.processChart = echarts.init(el) diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue index 220deda8..c9f25c89 100644 --- a/frontend/src/components/TableView/FieldsTableView.vue +++ b/frontend/src/components/TableView/FieldsTableView.vue @@ -83,7 +83,7 @@ @@ -93,6 +93,8 @@