Merge remote-tracking branch 'upstream/develop' into upstream-develop

# Conflicts:
#	backend/main.go
#	frontend/.env.development
This commit is contained in:
陈景阳
2020-01-28 15:51:09 +08:00
22 changed files with 305 additions and 88 deletions

View File

@@ -2,7 +2,7 @@
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
labels: 'bug'
assignees: ''
---
@@ -22,17 +22,3 @@ A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Smartphone (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Browser [e.g. stock browser, safari]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.

23
.github/ISSUE_TEMPLATE/bug_report_zh.md vendored Normal file
View File

@@ -0,0 +1,23 @@
---
name: Bug 报告
about: 创建一份 Bug 报告帮助我们优化产品
title: ''
labels: 'bug'
assignees: ''
---
**Bug 描述**
例如,当 xxx 时xxx 功能不工作。
**复现步骤**
该 Bug 复现步骤如下
1.
2.
3.
**期望结果**
xxx 能工作。
**截屏**
![截屏1](http://static-docs.crawlab.cn/login.png)

View File

@@ -2,7 +2,7 @@
name: Feature request
about: Suggest an idea for this project
title: ''
labels: ''
labels: 'enhancement'
assignees: ''
---
@@ -15,6 +15,3 @@ A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

View File

@@ -0,0 +1,17 @@
---
name: 功能需求
about: 优化和功能需求建议
title: ''
labels: 'enhancement'
assignees: ''
---
**请描述该需求尝试解决的问题**
例如,当 xxx 时,我总是被当前 xxx 的设计所困扰。
**请描述您认为可行的解决方案**
例如,添加 xxx 功能能够解决问题。
**考虑过的替代方案**
例如,如果用 xxx也能解决该问题。

View File

@@ -133,6 +133,8 @@ func main() {
anonymousGroup.POST("/login", routes.Login) // 用户登录
anonymousGroup.PUT("/users", routes.PutUser) // 添加用户
anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息
// release版本
anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本
}
authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
{

View File

@@ -27,7 +27,7 @@ services:
# CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址
# CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口
# CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱
# CRAWLAB_NOTIFICATION_MAIL_SENDEREIDENTITY: admin@exmaple.com # sender ID 发送者 ID
# CRAWLAB_NOTIFICATION_MAIL_SENDERIDENTITY: admin@exmaple.com # sender ID 发送者 ID
# CRAWLAB_NOTIFICATION_MAIL_SMTP_USER: username # SMTP username SMTP 用户名
# CRAWLAB_NOTIFICATION_MAIL_SMTP_PASSWORD: password # SMTP password SMTP 密码
ports:

View File

@@ -1,4 +1,3 @@
NODE_ENV='development'
VUE_APP_BASE_URL=http://localhost:8000
VUE_APP_CRAWLAB_BASE_URL=http://api.crawlab.cn
VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn

View File

@@ -1,3 +1,3 @@
NODE_ENV='production'
VUE_APP_BASE_URL=/api
VUE_APP_CRAWLAB_BASE_URL=http://api.crawlab.cn
VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn

View File

@@ -1,3 +1,3 @@
NODE_ENV='test'
VUE_APP_BASE_URL='http://localhost:8000'
VUE_APP_CRAWLAB_BASE_URL=http://api.crawlab.cn
VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn

View File

@@ -26,6 +26,7 @@
"echarts": "^4.1.0",
"element-ui": "2.13.0",
"font-awesome": "^4.7.0",
"github-markdown-css": "^3.0.1",
"js-cookie": "2.2.0",
"normalize.css": "7.0.0",
"nprogress": "0.2.0",

View File

@@ -34,12 +34,8 @@ export default {
}
},
methods: {},
created () {
this.$store.dispatch('setting/getSetting')
},
mounted () {
async mounted () {
window.setUseStats = (value) => {
localStorage.setItem('useStats', value)
document.querySelector('.el-message__closeBtn').click()
if (value === 1) {
this.$st.sendPv('/allow_stats')
@@ -48,6 +44,7 @@ export default {
this.$st.sendPv('/disallow_stats')
this.$st.sendEv('全局', '允许/禁止统计', '禁止')
}
localStorage.setItem('useStats', value)
}
// first-time user

View File

@@ -30,7 +30,9 @@
</span>
</el-dialog>
<div class="file-tree-wrapper">
<div
class="file-tree-wrapper"
>
<el-tree
:data="computedFileTree"
ref="tree"
@@ -45,20 +47,20 @@
<span class="custom-tree-node" slot-scope="{ node, data }">
<el-popover v-model="isShowCreatePopoverDict[data.path]" trigger="manual" placement="right"
popper-class="create-item-popover" :visible-arrow="false" @hide="onHideCreate(data)">
<div class="create-item-title">
<span class="item-icon">
<font-awesome-icon icon="plus" color="rgba(3,47,98,.5)"/>
</span>
<span class="create-item-text">{{$t('Create')}}</span>
</div>
<ul class="create-item-list">
<li class="create-item" @click="dirDialogVisible = true">
<font-awesome-icon :icon="['fa', 'folder']" color="rgba(3,47,98,.5)"/>
<span class="create-item-text">{{$t('Directory')}}</span>
</li>
<li class="create-item" @click="fileDialogVisible = true">
<ul class="action-item-list">
<li class="action-item" @click="fileDialogVisible = true">
<font-awesome-icon icon="file-alt" color="rgba(3,47,98,.5)"/>
<span class="create-item-text">{{$t('File')}}</span>
<span class="action-item-text">{{$t('Create File')}}</span>
</li>
<li class="action-item" @click="dirDialogVisible = true">
<font-awesome-icon :icon="['fa', 'folder']" color="rgba(3,47,98,.5)"/>
<span class="action-item-text">{{$t('Create Directory')}}</span>
</li>
</ul>
<ul class="action-item-list">
<li class="action-item" @click="onClickRemoveNav(data)">
<font-awesome-icon :icon="['fa', 'trash']" color="rgba(3,47,98,.5)"/>
<span class="action-item-text">{{$t('Remove')}}</span>
</li>
</ul>
<template slot="reference">
@@ -85,11 +87,38 @@
</el-popover>
</span>
</el-tree>
<el-popover trigger="click" placement="right"
popper-class="create-item-popover" :visible-arrow="false">
<ul class="action-item-list">
<li class="action-item" @click="fileDialogVisible = true">
<font-awesome-icon icon="file-alt" color="rgba(3,47,98,.5)"/>
<span class="action-item-text">{{$t('Create File')}}</span>
</li>
<li class="action-item" @click="dirDialogVisible = true">
<font-awesome-icon :icon="['fa', 'folder']" color="rgba(3,47,98,.5)"/>
<span class="action-item-text">{{$t('Create Directory')}}</span>
</li>
</ul>
<div
class="add-btn-wrapper"
slot="reference"
>
<el-button
class="add-btn"
size="small"
type="primary"
icon="el-icon-plus"
@click="onEmptyClick"
>
{{$t('Add')}}
</el-button>
</div>
</el-popover>
</div>
<div class="main-content">
<div v-if="!showFile" class="file-list">
{{$t('Please select a file on the left.')}}
{{$t('Please select a file or click the add button on the left.')}}
</div>
<template v-else>
<div class="top-part">
@@ -172,7 +201,8 @@ export default {
activeFileNode: {},
dirDialogVisible: false,
fileDialogVisible: false,
nodeExpandedDict: {}
nodeExpandedDict: {},
isShowDeleteNav: false
}
},
computed: {
@@ -349,9 +379,34 @@ export default {
this.activeFileNode = data
this.$st.sendEv('爬虫详情', '文件', '右键点击导航栏')
},
onEmptyClick () {
const data = { path: '' }
this.isShowCreatePopoverDict = {}
this.$set(this.isShowCreatePopoverDict, data.path, true)
this.activeFileNode = data
this.$st.sendEv('爬虫详情', '文件', '空白点击添加')
},
onHideCreate (data) {
this.$set(this.isShowCreatePopoverDict, data.path, false)
this.name = ''
},
onClickRemoveNav (data) {
this.$confirm(this.$t('Are you sure to delete this file/directory?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel'),
confirmButtonClass: 'danger',
type: 'warning'
}).then(() => {
this.onFileDeleteNav(data.path)
})
},
async onFileDeleteNav (path) {
await this.$store.dispatch('file/deleteFile', { path })
await this.$store.dispatch('spider/getFileTree')
this.$message.success(this.$t('Deleted successfully'))
this.isShowDelete = false
this.showFile = false
this.$st.sendEv('爬虫详情', '文件', '删除')
}
},
async created () {
@@ -509,33 +564,52 @@ export default {
margin: 0;
}
.create-item-list {
.action-item-list {
list-style: none;
padding: 0;
margin: 0
}
.create-item-title {
.action-item-title {
padding-top: 10px;
padding-left: 15px;
padding-bottom: 10px;
border-bottom: 1px solid #eaecef;
padding-bottom: 5px;
}
.create-item-list .create-item {
.action-item-list .action-item {
display: flex;
align-items: center;
height: 35px;
padding: 0 0 0 15px;
padding: 0 0 0 10px;
margin: 0;
cursor: pointer;
}
.create-item-list .create-item:hover {
.action-item-list .action-item:last-child {
border-bottom: 1px solid #eaecef;
}
.action-item-list .action-item:hover {
background: #F5F7FA;
}
.create-item-list .create-item .create-item-text {
.action-item-list .action-item svg {
width: 20px;
}
.action-item-list .action-item .action-item-text {
margin-left: 5px;
}
.add-btn-wrapper {
width: 220px;
border-top: 1px solid #eaecef;
margin: 10px 10px;
}
.add-btn-wrapper .add-btn {
width: 80px;
margin-left: calc(120px - 40px - 10px);
margin-top: 20px;
}
</style>

View File

@@ -1 +1,24 @@
export default {}
export default {
// 内容
addNodeInstruction: `
You cannot add nodes directly on the web interface in Crawlab.
Adding a node is quite simple. The only thing you have to do is to run a Crawlab service on your target machine.
#### Docker Deployment
If you are running Crawlab using Docker, you can start a new \`worker\` container on the target machine, or add a \`worker\` service in the \`docker-compose.yml\`.
\`\`\`bash
docker run -d --restart always --name crawlab_worker \\
-e CRAWLAB_SERVER_MASTER=N \\
-e CRAWLAB_MONGO_HOST=xxx.xxx.xxx.xxx \\ # make sure you are connecting to the same MongoDB
-e CRAWLAB_REDIS_ADDRESS=xxx.xxx.xxx.xxx \\ # make sure you are connecting to the same Redis
tikazyq/crawlab:latest
\`\`\`
#### Direct Deploy
If you are deploying directly, the only thing you have to do is to run a backend service on the target machine, you can refer to [Direct Deploy](https://docs.crawlab.cn/Installation/Direct.html).
For more information, please refer to the [Official Documentation](https://docs.crawlab.cn).
`
}

View File

@@ -68,6 +68,9 @@ export default {
'Rename': '重命名',
'Install': '安装',
'Uninstall': '卸载',
'Create Directory': '新建目录',
'Create File': '新建文件',
'Add Node': '添加节点',
// 主页
'Total Tasks': '总任务数',
@@ -277,6 +280,7 @@ export default {
'Notification': '提示',
'Are you sure to delete this node?': '你确定要删除该节点?',
'Are you sure to run this spider?': '你确定要运行该爬虫?',
'Are you sure to delete this file/directory?': '你确定要删除该文件/文件夹?',
'Added spider successfully': '成功添加爬虫',
'Uploaded spider files successfully': '成功上传爬虫文件',
'Node info has been saved successfully': '节点信息已成功保存',
@@ -325,7 +329,7 @@ export default {
'The schedule has been added': '已添加定时任务',
'The schedule has been saved': '已保存定时任务',
'Email format invalid': '邮箱地址格式不正确',
'Please select a file on the left.': '请在左侧选择一个文件.',
'Please select a file or click the add button on the left.': '请在左侧选择一个文件或点击添加按钮.',
'New Directory': '新建目录',
'Enter new directory name': '输入新目录名称',
'New directory name': '新目录名称',
@@ -384,6 +388,29 @@ export default {
cron_format: 'Cron 格式: [秒] [分] [小时] [日] [月] [周]'
},
// 内容
addNodeInstruction: `
您不能在 Crawlab 的 Web 界面直接添加节点。
添加节点的方式非常简单,您只需要在目标机器上运行一个 Crawlab 服务就可以了。
#### Docker 部署
如果您是用 Docker 启动 Crawlab可以在目标机器上运行一个新的 \`worker\` 容器,或者在 \`docker-compose.yml\` 中添加 \`worker\` 服务。
\`\`\`bash
docker run -d --restart always --name crawlab_worker \\
-e CRAWLAB_SERVER_MASTER=N \\
-e CRAWLAB_MONGO_HOST=xxx.xxx.xxx.xxx \\ # 保证连接的是同一个 MongoDB
-e CRAWLAB_REDIS_ADDRESS=xxx.xxx.xxx.xxx \\ # 保证连接的是同一个 Redis
tikazyq/crawlab:latest
\`\`\`
#### 直接部署
如果您是用直接部署,只需要在目标机器上启动一个后端服务,请参考 [直接部署文档](https://docs.crawlab.cn/Installation/Direct.html)。
更多信息,请参考 [官方文档](https://docs.crawlab.cn)。
`,
// 其他
'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'
}

View File

@@ -1,6 +1,8 @@
import Vue from 'vue'
import Router from 'vue-router'
import store from '../store'
import request from '../api/request'
import stats from '../utils/stats'
/* Layout */
@@ -257,8 +259,13 @@ router.beforeEach((to, from, next) => {
}
})
router.afterEach((to, from, next) => {
router.afterEach(async (to, from, next) => {
if (to.path) {
await store.dispatch('setting/getSetting')
const res = await request.get('/version')
const version = res.data.data
store.commit('version/SET_VERSION', version)
sessionStorage.setItem('v', version)
stats.sendPv(to.path)
}
})

View File

@@ -14,6 +14,7 @@ import lang from './modules/lang'
import site from './modules/site'
import stats from './modules/stats'
import setting from './modules/setting'
import version from './modules/version'
import getters from './getters'
Vue.use(Vuex)
@@ -33,7 +34,8 @@ const store = new Vuex.Store({
lang,
site,
setting,
// 百度统计
version,
// 统计
stats
},
getters

View File

@@ -0,0 +1,21 @@
const state = {
version: ''
}
const getters = {}
const mutations = {
SET_VERSION: (state, value) => {
state.version = value
}
}
const actions = {}
export default {
namespaced: true,
state,
getters,
mutations,
actions
}

View File

@@ -47,17 +47,16 @@ export default {
},
isCollapse () {
return !this.sidebar.opened
},
version () {
return this.$store.state.version.version || window.sessionStorage.getItem('v')
}
},
data () {
return {
version: ''
}
},
async created () {
const res = await this.$request.get('/version')
this.version = res.data.data
sessionStorage.setItem('v', this.version)
}
}
</script>

View File

@@ -1,26 +1,31 @@
<template>
<div class="app-container">
<!--filter-->
<div v-if="false" class="filter">
<el-input prefix-icon="el-icon-search"
:placeholder="$t('Search')"
class="filter-search"
v-model="filter.keyword"
@change="onSearch">
</el-input>
<div class="right">
<el-button type="success"
icon="el-icon-refresh"
class="refresh"
@click="onRefresh">
{{$t('Refresh')}}
</el-button>
<el-dialog
:visible.sync="isShowAddNodeInstruction"
:title="$t('Notification')"
width="720px"
>
<div
v-html="addNodeInstructionHtml"
class="content markdown-body"
>
</div>
</div>
<!--./filter-->
</el-dialog>
<el-tabs type="border-card" v-model="activeTab">
<el-tab-pane :label="$t('Node List')">
<!--filter-->
<div class="filter-wrapper">
<el-button
size="small"
type="success"
icon="el-icon-plus"
@click="onAddNode"
>
{{$t('Add Node')}}
</el-button>
</div>
<!--./filter-->
<!--table list-->
<el-table :data="filteredTableData"
class="table"
@@ -115,7 +120,8 @@
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
</el-tooltip>
<el-tooltip :content="$t('Remove')" placement="top">
<el-button v-if="scope.row.status !== 'online'" type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
<el-button v-if="scope.row.status !== 'online'" type="danger" icon="el-icon-delete" size="mini"
@click="onRemove(scope.row)"></el-button>
</el-tooltip>
</template>
</el-table-column>
@@ -141,9 +147,11 @@
</template>
<script>
import showdown from 'showdown'
import {
mapState
} from 'vuex'
import 'github-markdown-css/github-markdown.css'
import NodeNetwork from '../../components/Node/NodeNetwork'
export default {
@@ -172,7 +180,11 @@ export default {
nodeFormRules: {
name: [{ required: true, message: 'Required Field', trigger: 'change' }]
},
activeTab: undefined
activeTab: undefined,
isButtonClicked: false,
isShowAddNodeInstruction: false,
converter: new showdown.Converter(),
addNodeInstructionMarkdown: 'addNodeInstruction'
}
},
computed: {
@@ -191,16 +203,23 @@ export default {
}
return false
})
},
addNodeInstructionHtml () {
if (!this.converter) return
return this.converter.makeHtml(this.$t(this.addNodeInstructionMarkdown))
}
},
methods: {
onSearch () {
},
onAdd () {
this.$store.commit('node/SET_NODE_FORM', [])
this.isEditMode = false
this.dialogVisible = true
onAddNode () {
this.isShowAddNodeInstruction = true
},
// onAdd () {
// this.$store.commit('node/SET_NODE_FORM', [])
// this.isEditMode = false
// this.dialogVisible = true
// },
onRefresh () {
this.$store.dispatch('node/getNodeList')
this.$st.sendEv('节点列表', '刷新')
@@ -235,6 +254,11 @@ export default {
this.dialogVisible = true
},
onRemove (row) {
this.isButtonClicked = true
setTimeout(() => {
this.isButtonClicked = false
}, 100)
this.$confirm(this.$t('Are you sure to delete this node?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel'),
@@ -251,6 +275,11 @@ export default {
})
},
onView (row) {
this.isButtonClicked = true
setTimeout(() => {
this.isButtonClicked = false
}, 100)
this.$router.push(`/nodes/${row._id}`)
this.$st.sendEv('节点列表', '查看节点')
@@ -262,6 +291,7 @@ export default {
this.$store.dispatch('node/getNodeSystemInfo', row._id)
},
onRowClick (row) {
if (this.isButtonClicked) return
this.onView(row)
},
getExecutables (row) {
@@ -312,6 +342,13 @@ export default {
padding: 7px;
}
.filter-wrapper {
text-align: right;
}
.content {
word-break: break-word;
}
</style>
<style>
.node-detail .el-form-item {

View File

@@ -118,9 +118,9 @@ export default {
await this.$store.dispatch('spider/getSpiderList')
// if spider is configurable spider, set to config tab by default
if (this.spiderForm.type === 'configurable') {
this.activeTabName = 'config'
}
// if (this.spiderForm.type === 'configurable') {
// this.activeTabName = 'config'
// }
}
}
</script>

View File

@@ -3946,6 +3946,11 @@ github-buttons@^2.3.0:
resolved "https://registry.npm.taobao.org/github-buttons/download/github-buttons-2.6.0.tgz#fa3e031451cee7ba05c3254fa67c73fe783104dc"
integrity sha1-+j4DFFHO57oFwyVPpnxz/ngxBNw=
github-markdown-css@^3.0.1:
version "3.0.1"
resolved "https://registry.npm.taobao.org/github-markdown-css/download/github-markdown-css-3.0.1.tgz#d08db1060d2e182025e0d07d547cfe2afed30205"
integrity sha1-0I2xBg0uGCAl4NB9VHz+Kv7TAgU=
glob-base@^0.3.0:
version "0.3.0"
resolved "http://registry.npm.taobao.org/glob-base/download/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4"

View File

@@ -3,7 +3,7 @@ services:
master:
image: "tikazyq/crawlab:master"
environment:
CRAWLAB_API_ADDRESS: "http://crawlab.cn/api"
CRAWLAB_API_ADDRESS: "https://crawlab.cn/api"
CRAWLAB_BASE_URL: "/demo"
CRAWLAB_SERVER_MASTER: "Y"
CRAWLAB_MONGO_HOST: "mongo"