diff --git a/README-zh.md b/README-zh.md index 43b441a6..4b662f23 100644 --- a/README-zh.md +++ b/README-zh.md @@ -2,7 +2,7 @@

- + diff --git a/README.md b/README.md index 636bd198..0ca2a772 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

- + diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 321860a2..32fce297 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -38,6 +38,7 @@ other: version: 0.4.7 setting: allowRegister: "N" + enableTutorial: "N" notification: mail: server: '' diff --git a/backend/entity/doc.go b/backend/entity/doc.go new file mode 100644 index 00000000..b356d38a --- /dev/null +++ b/backend/entity/doc.go @@ -0,0 +1,8 @@ +package entity + +type DocItem struct { + Title string `json:"title"` + Url string `json:"url"` + Path string `json:"path"` + Children []DocItem `json:"children"` +} diff --git a/backend/main.go b/backend/main.go index a35291ca..ea9c30df 100644 --- a/backend/main.go +++ b/backend/main.go @@ -135,6 +135,8 @@ func main() { // release版本 anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本 anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本 + // 文档 + anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据 } authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) { @@ -259,7 +261,6 @@ func main() { authGroup.GET("/git/branches", routes.GetGitBranches) // 获取 Git 分支 authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 } - } // 路由ping diff --git a/backend/routes/doc.go b/backend/routes/doc.go new file mode 100644 index 00000000..6426cdcc --- /dev/null +++ b/backend/routes/doc.go @@ -0,0 +1,25 @@ +package routes + +import ( + "crawlab/services" + "github.com/apex/log" + "github.com/gin-gonic/gin" + "net/http" + "runtime/debug" +) + +func GetDocs(c *gin.Context) { + type ResData struct { + String string `json:"string"` + } + data, err := services.GetDocs() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: ResData{String:data}, + }) +} diff --git a/backend/routes/setting.go b/backend/routes/setting.go index 4429873e..83976455 100644 --- a/backend/routes/setting.go +++ b/backend/routes/setting.go @@ -7,7 +7,8 @@ import ( ) type SettingBody struct { - AllowRegister string `json:"allow_register"` + AllowRegister string `json:"allow_register"` + EnableTutorial string `json:"enable_tutorial"` } func GetVersion(c *gin.Context) { @@ -21,9 +22,10 @@ func GetVersion(c *gin.Context) { } func GetSetting(c *gin.Context) { - allowRegister := viper.GetString("setting.allowRegister") - - body := SettingBody{AllowRegister: allowRegister} + body := SettingBody{ + AllowRegister: viper.GetString("setting.allowRegister"), + EnableTutorial: viper.GetString("setting.enableTutorial"), + } c.JSON(http.StatusOK, Response{ Status: "ok", diff --git a/backend/services/doc.go b/backend/services/doc.go new file mode 100644 index 00000000..572e5cb4 --- /dev/null +++ b/backend/services/doc.go @@ -0,0 +1,27 @@ +package services + +import ( + "github.com/apex/log" + "github.com/imroc/req" + "runtime/debug" +) + +func GetDocs() (data string, err error) { + // 获取远端数据 + res, err := req.Get("https://docs.crawlab.cn/search_plus_index.json") + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return data, err + } + + // 反序列化 + data, err = res.ToString() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return data, err + } + + return data, nil +} diff --git a/docker-compose.yml b/docker-compose.yml index 9d5acbb3..affdf7bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: # CRAWLAB_TASK_WORKERS: 4 # number of task executors 任务执行器个数（并行执行任务数） # CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境 # CRAWLAB_SETTING_ALLOWREGISTER: "N" # whether to allow user registration 是否允许用户注册 + # CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程 # CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址 # CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口 # CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱 @@ -47,6 +48,9 @@ services: depends_on: - mongo - redis + # environment: + # MONGO_INITDB_ROOT_USERNAME: username + # MONGO_INITDB_ROOT_PASSWORD: password # volumes: # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 mongo: diff --git a/frontend/.env.development b/frontend/.env.development index ba21c0c6..275af85c 100644 --- a/frontend/.env.development +++ b/frontend/.env.development @@ -1,3 +1,4 @@ NODE_ENV='development' VUE_APP_BASE_URL=http://localhost:8000 -VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn \ No newline at end of file +VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn +VUE_APP_DOC_URL=http://docs.crawlab.cn diff --git a/frontend/.env.production b/frontend/.env.production index d01edd25..7cca0821 100644 --- a/frontend/.env.production +++ b/frontend/.env.production @@ -1,3 +1,4 @@ NODE_ENV='production' VUE_APP_BASE_URL=/api VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn +VUE_APP_DOC_URL=http://docs.crawlab.cn diff --git a/frontend/.env.test b/frontend/.env.test index e9aeafc9..f29b1cd3 100644 --- a/frontend/.env.test +++ b/frontend/.env.test @@ -1,3 +1,4 @@ NODE_ENV='test' VUE_APP_BASE_URL='http://localhost:8000' VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn +VUE_APP_DOC_URL=http://docs.crawlab.cn diff --git a/frontend/src/components/Documentation/Documentation.vue b/frontend/src/components/Documentation/Documentation.vue new file mode 100644 index 00000000..9abde012 --- /dev/null +++ b/frontend/src/components/Documentation/Documentation.vue @@ -0,0 +1,114 @@ + + + + diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index e275afd0..f02ca169 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -448,6 +448,11 @@ export default { 'Key': '设置', 'Allow Sending Statistics': '允许发送统计信息', 'General': '通用', + 'Enable Tutorial': '启用教程', + + // 全局 + 'Related Documentation': '相关文档', + 'Click to view related Documentation': '点击查看相关文档', // 其他 tagsView: { diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index 4fcb86db..9afa6d40 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -17,6 +17,7 @@ import setting from './modules/setting' import version from './modules/version' import tour from './modules/tour' import project from './modules/project' +import doc from './modules/doc' import getters from './getters' Vue.use(Vuex) @@ -39,6 +40,7 @@ const store = new Vuex.Store({ version, tour, project, + doc, // 统计 stats }, diff --git a/frontend/src/store/modules/doc.js b/frontend/src/store/modules/doc.js new file mode 100644 index 00000000..4472e6e7 --- /dev/null +++ b/frontend/src/store/modules/doc.js @@ -0,0 +1,61 @@ +import request from '../../api/request' + +const state = { + docData: [] +} + +const getters = {} + +const mutations = { + SET_DOC_DATA (state, value) { + state.docData = value + } +} + +const actions = { + async getDocData ({ commit }) { + const res = await request.get('/docs') + + const data = JSON.parse(res.data.data.string) + + // init cache + const cache = {} + + // iterate paths + for (let path in data) { + if (data.hasOwnProperty(path)) { + const d = data[path] + if (path.match(/\/$/)) { + cache[path] = d + cache[path].children = [] + } else if (path.match(/\.html$/)) { + const parentPath = path.split('/')[0] + '/' + cache[parentPath].children.push(d) + } + } + } + + commit('SET_DOC_DATA', Object.values(cache).map(d => { + d.level = 1 + d.label = d.title + d.fullUrl = process.env.VUE_APP_DOC_URL + '/' + d.url + if (d.children) { + d.children = d.children.map(c => { + c.level = 2 + c.label = c.title + c.fullUrl = process.env.VUE_APP_DOC_URL + '/' + c.url + return c + }) + } + return d + })) + } +} + +export default { + namespaced: true, + state, + getters, + mutations, + actions +} diff --git a/frontend/src/store/modules/setting.js b/frontend/src/store/modules/setting.js index a44f7c9b..29732e2b 100644 --- a/frontend/src/store/modules/setting.js +++ b/frontend/src/store/modules/setting.js @@ -16,6 +16,12 @@ const actions = { async getSetting ({ commit }) { const res = await request.get('/setting') commit('SET_SETTING', res.data.data) + + // set default enable_tutorial + const enableTutorial = res.data.data.enable_tutorial + if (!localStorage.getItem('enableTutorial')) { + localStorage.setItem('enableTutorial', enableTutorial === 'Y' ? '1' : '0') + } } } diff --git a/frontend/src/utils/doc.js b/frontend/src/utils/doc.js new file mode 100644 index 00000000..20983988 --- /dev/null +++ b/frontend/src/utils/doc.js @@ -0,0 +1,28 @@ +export default { + docs: [ + { + path: '/projects', + pattern: '^Project' + }, + { + path: '/spiders', + pattern: '^Spider|^SDK|^Integration|^CI/Git' + }, + { + path: '/tasks', + pattern: '^Task|^Architecture/Task' + }, + { + path: '/schedules', + pattern: '^Schedule' + }, + { + path: '/nodes', + pattern: '^Node|^Architecture/Node' + }, + { + path: '/setting', + pattern: '^Notification' + } + ] +} diff --git a/frontend/src/utils/index.js b/frontend/src/utils/index.js index c371f672..765d0e93 100644 --- a/frontend/src/utils/index.js +++ b/frontend/src/utils/index.js @@ -3,11 +3,13 @@ import encrypt from './encrypt' import tour from './tour' import log from './log' import scrapy from './scrapy' +import doc from './doc' export default { stats, encrypt, tour, log, - scrapy + scrapy, + doc } diff --git a/frontend/src/utils/tour.js b/frontend/src/utils/tour.js index b974d692..33945603 100644 --- a/frontend/src/utils/tour.js +++ b/frontend/src/utils/tour.js @@ -18,6 +18,10 @@ export default { } return !!data[tourName] }, + startTour: (vm, tourName) => { + if (localStorage.getItem('enableTutorial') === '0') return + vm.$tours[tourName].start() + }, finishTour: (tourName) => { let data try { diff --git a/frontend/src/views/layout/Layout.vue b/frontend/src/views/layout/Layout.vue index 119c7337..8576be57 100644 --- a/frontend/src/views/layout/Layout.vue +++ b/frontend/src/views/layout/Layout.vue @@ -1,12 +1,36 @@ @@ -18,16 +42,23 @@ import { TagsView } from './components' import ResizeMixin from './mixin/ResizeHandler' +import Documentation from '../../components/Documentation/Documentation' export default { name: 'Layout', components: { + Documentation, Navbar, Sidebar, TagsView, AppMain }, mixins: [ResizeMixin], + data () { + return { + isShowDocumentation: false + } + }, computed: { sidebar () { return this.$store.state.app.sidebar @@ -47,7 +78,18 @@ export default { methods: { handleClickOutside () { this.$store.dispatch('CloseSideBar', { withoutAnimation: false }) + }, + onClickDocumentation () { + this.isShowDocumentation = true + this.$st.sendEv('全局', '打开右侧文档') + }, + onCloseDocumentation () { + this.isShowDocumentation = false + this.$st.sendEv('全局', '关闭右侧文档') } + }, + async created () { + await this.$store.dispatch('doc/getDocData') } } @@ -77,4 +119,45 @@ export default { position: absolute; z-index: 999; } + + .documentation { + z-index: 9999; + position: fixed; + right: 25px; + bottom: 20px; + font-size: 32px; + cursor: pointer; + color: #909399; + } + + + diff --git a/frontend/src/views/node/NodeDetail.vue b/frontend/src/views/node/NodeDetail.vue index 24b81b63..7f87a87e 100644 --- a/frontend/src/views/node/NodeDetail.vue +++ b/frontend/src/views/node/NodeDetail.vue @@ -124,7 +124,7 @@ export default { }, mounted () { if (!this.$utils.tour.isFinishedTour('node-detail')) { - this.$tours['node-detail'].start() + this.$utils.tour.startTour(this, 'node-detail') this.$st.sendEv('教程', '开始', 'node-detail') } } diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 7308311c..adbfef56 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -467,7 +467,7 @@ export default { if (!this.$utils.tour.isFinishedTour('schedule-list-add')) { setTimeout(() => { - this.$tours['schedule-list-add'].start() + this.$utils.tour.startTour(this, 'schedule-list-add') this.$st.sendEv('教程', '开始', 'schedule-list-add') }, 500) } @@ -617,7 +617,7 @@ export default { mounted () { if (!this.isDisabledSpiderSchedule) { if (!this.$utils.tour.isFinishedTour('schedule-list')) { - this.$tours['schedule-list'].start() + this.$utils.tour.startTour(this, 'schedule-list') this.$st.sendEv('教程', '开始', 'schedule-list') } } diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue index c341ccb7..86118e71 100644 --- a/frontend/src/views/setting/Setting.vue +++ b/frontend/src/views/setting/Setting.vue @@ -9,7 +9,7 @@ /> - + @@ -30,6 +30,7 @@ + @@ -49,6 +50,14 @@ inactive-color="#909399" /> + + +

@@ -213,7 +222,8 @@ export default { this.$utils.tour.nextStep('setting', currentStep) } }, - isAllowSendingStatistics: localStorage.getItem('useStats') === '1' + isAllowSendingStatistics: localStorage.getItem('useStats') === '1', + isEnableTutorial: localStorage.getItem('enableTutorial') === '1' } }, computed: { @@ -291,6 +301,10 @@ export default { } this.$message.success(this.$t('Saved successfully')) localStorage.setItem('useStats', value ? '1' : '0') + }, + onEnableTutorialChange (value) { + this.$message.success(this.$t('Saved successfully')) + localStorage.setItem('enableTutorial', value ? '1' : '0') } }, async created () { @@ -300,7 +314,7 @@ export default { }, mounted () { if (!this.$utils.tour.isFinishedTour('setting')) { - this.$tours['setting'].start() + this.$utils.tour.startTour(this, 'setting') this.$st.sendEv('教程', '开始', 'setting') } } diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index 787788d3..a69545b2 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -208,7 +208,7 @@ export default { if (!this.$utils.tour.isFinishedTour('spider-detail-config')) { setTimeout(() => { - this.$tours['spider-detail-config'].start() + this.$utils.tour.startTour(this, 'spider-detail-config') this.$st.sendEv('教程', '开始', 'spider-detail-config') }, 100) } @@ -260,7 +260,7 @@ export default { }, mounted () { if (!this.$utils.tour.isFinishedTour('spider-detail')) { - this.$tours['spider-detail'].start() + this.$utils.tour.startTour(this, 'spider-detail') this.$st.sendEv('教程', '开始', 'spider-detail') } } diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index a8996b8e..8e6a6ba9 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -887,7 +887,7 @@ export default { setTimeout(() => { if (!this.$utils.tour.isFinishedTour('spider-list-add')) { - this.$tours['spider-list-add'].start() + this.$utils.tour.startTour(this, 'spider-list-add') this.$st.sendEv('教程', '开始', 'spider-list-add') } }, 300) @@ -1241,7 +1241,7 @@ export default { }) if (!this.$utils.tour.isFinishedTour('spider-list')) { - this.$tours['spider-list'].start() + this.$utils.tour.startTour(this, 'spider-list') this.$st.sendEv('教程', '开始', 'spider-list') } }, diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index 11051ed0..422dc088 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -202,7 +202,7 @@ export default { }, mounted () { if (!this.$utils.tour.isFinishedTour('task-detail')) { - this.$tours['task-detail'].start() + this.$utils.tour.startTour(this, 'task-detail') this.$st.sendEv('教程', '开始', 'task-detail') } }, diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index 98b720b3..3cd564d9 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -412,7 +412,7 @@ export default { }, 5000) if (!this.$utils.tour.isFinishedTour('task-list')) { - this.$tours['task-list'].start() + this.$utils.tour.startTour(this, 'task-list') this.$st.sendEv('教程', '开始', 'task-list') } }, diff --git a/k8s/crawlab-master.yaml b/k8s/crawlab-master.yaml new file mode 100644 index 00000000..b87f6881 --- /dev/null +++ b/k8s/crawlab-master.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Service +metadata: + name: crawlab + namespace: crawlab +spec: + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30088 + selector: + app: crawlab-master + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-master + namespace: crawlab +spec: + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-master + template: + metadata: + labels: + app: crawlab-master + spec: + containers: + - image: tikazyq/crawlab:latest + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "Y" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + ports: + - containerPort: 8080 + name: crawlab diff --git a/k8s/crawlab-worker.yaml b/k8s/crawlab-worker.yaml new file mode 100644 index 00000000..b54a7796 --- /dev/null +++ b/k8s/crawlab-worker.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-worker + namespace: crawlab +spec: + replicas: 4 + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-worker + template: + metadata: + labels: + app: crawlab-worker + spec: + containers: + - image: tikazyq/crawlab:latest + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "N" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" diff --git a/k8s/mongo-pv.yaml b/k8s/mongo-pv.yaml new file mode 100644 index 00000000..ad914131 --- /dev/null +++ b/k8s/mongo-pv.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: mongo-pv-volume + namespace: crawlab + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data/k8s/mongodb/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-pv-claim + namespace: crawlab +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi \ No newline at end of file diff --git a/k8s/mongo.yaml b/k8s/mongo.yaml new file mode 100644 index 00000000..9b5bd3b6 --- /dev/null +++ b/k8s/mongo.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongo + namespace: crawlab +spec: + ports: + - port: 27017 + selector: + app: mongo + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: mongo + namespace: crawlab +spec: + selector: + matchLabels: + app: mongo + strategy: + type: Recreate + template: + metadata: + labels: + app: mongo + spec: + containers: + - image: mongo:4 + name: mongo + ports: + - containerPort: 27017 + name: mongo + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + volumes: + - name: mongo-persistent-storage + persistentVolumeClaim: + claimName: mongo-pv-claim \ No newline at end of file diff --git a/k8s/redis.yaml b/k8s/redis.yaml new file mode 100644 index 00000000..49df8dde --- /dev/null +++ b/k8s/redis.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: crawlab +spec: + ports: + - port: 6379 + selector: + app: redis + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: redis + namespace: crawlab +spec: + selector: + matchLabels: + app: redis + strategy: + type: Recreate + template: + metadata: + labels: + app: redis + spec: + containers: + - image: redis + name: redis + ports: + - containerPort: 6379 + name: redis \ No newline at end of file