From 52a8fce89498efed8d9955bd8c8a52f94136111c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Thu, 2 Jan 2020 10:07:21 +0800 Subject: [PATCH 01/27] =?UTF-8?q?=E4=BC=98=E5=8C=96UI=E7=95=8C=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/node.go | 30 +++++++++++++++++-- .../components/InfoView/SpiderInfoView.vue | 6 ++-- frontend/src/views/schedule/ScheduleList.vue | 2 +- frontend/src/views/spider/SpiderList.vue | 11 +++---- frontend/src/views/task/TaskList.vue | 4 +-- 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/backend/services/node.go b/backend/services/node.go index d14ce4ae..47d6185a 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -102,7 +102,7 @@ func UpdateNodeStatus() { model.ResetNodeStatusToOffline(list) } -// 处理接到信息 +// 处理节点信息 func handleNodeInfo(key string, data *Data) { // 添加同步锁 v, err := database.RedisClient.Lock(key) @@ -168,8 +168,32 @@ func UpdateNodeData() { return } + // 构造节点数据 + data := Data{ + Key: key, + Mac: mac, + Ip: ip, + Master: model.IsMaster(), + UpdateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + } + + // 注册节点到Redis + dataBytes, err := json.Marshal(&data) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } + + if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil { + log.Errorf(err.Error()) + return + } + + // 注释掉，无需这样处理。 直接覆盖key对应的节点信息即可 by xyz 2020.01.01 //先获取所有Redis的nodekey - list, _ := database.RedisClient.HKeys("nodes") + /*list, _ := database.RedisClient.HKeys("nodes") if i := utils.Contains(list, key); i == false { // 构造节点数据 @@ -193,7 +217,7 @@ func UpdateNodeData() { log.Errorf(err.Error()) return } - } + }*/ } diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index a13b4c7a..df3864e1 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -61,15 +61,15 @@ :file-list="fileList" style="display:inline-block;margin-right:10px" > - + {{$t('Upload')}} - {{$t('Run')}} - {{$t('Save')}} diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 23d9fa97..0d1ff0b5 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -143,7 +143,7 @@ - + diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 80c3fa0b..18620944 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -34,6 +34,7 @@ :visible.sync="addDialogVisible" :before-close="onAddDialogClose"> + @@ -55,8 +56,7 @@ :headers="{Authorization:token}" :on-success="onUploadSuccess" :file-list="fileList"> - + {{$t('Upload')}} @@ -75,9 +75,10 @@

- {{$t('Add')}} + {{$t('Add')}}
+ @@ -101,7 +102,7 @@
- {{$t('Add')}} + {{$t('Add')}}
@@ -146,7 +147,7 @@ {{$t('Import Spiders')}}
- + @@ -125,7 +125,7 @@ :width="col.width"> - + @@ -38,10 +41,12 @@ import SpiderOverview from '../../components/Overview/SpiderOverview' import EnvironmentList from '../../components/Environment/EnvironmentList' import SpiderStats from '../../components/Stats/SpiderStats' import ConfigList from '../../components/Config/ConfigList' +import SpiderSchedules from './SpiderSchedules' export default { name: 'SpiderDetail', components: { + SpiderSchedules, ConfigList, SpiderStats, EnvironmentList, diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 80c3fa0b..fa6c6718 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -556,11 +556,6 @@ export default { } this.$store.dispatch('spider/getSpiderList', params) } - // getTypes () { - // request.get(`/spider/types`).then(resp => { - // this.types = resp.data.data - // }) - // } }, async created () { // fetch spider types diff --git a/frontend/src/views/spider/SpiderSchedules.vue b/frontend/src/views/spider/SpiderSchedules.vue new file mode 100644 index 00000000..0315e47d --- /dev/null +++ b/frontend/src/views/spider/SpiderSchedules.vue @@ -0,0 +1,55 @@ + From b98240fab6009d65fb11dc183876fbd3b7ffe5ac Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 3 Jan 2020 17:24:57 +0800 Subject: [PATCH 03/27] updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1275b9fa..81ea730d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Features / Enhancement - **Dependency Installation**. Allow users to install/uninstall dependencies and add programming languages (Node.js only for now) on the platform web interface. - **Pre-install Programming Languages in Docker**. Allow Docker users to set `CRAWLAB_SERVER_LANG_NODE` as `Y` to pre-install `Node.js` environments. +- **Add Schedule List in Spider Detail Page**. Allow users to view / add / edit schedule cron jobs in the spider detail page. [#360](https://github.com/crawlab-team/crawlab/issues/360) # 0.4.2 (2019-12-26) From c88c3abf85ec52af82267b6680535e4ca8045eb2 Mon Sep 17 00:00:00 2001 From: hantmac Date: Sat, 4 Jan 2020 17:29:05 +0800 Subject: [PATCH 04/27] Add Unique index for 'key' --- backend/database/mongo.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/database/mongo.go b/backend/database/mongo.go index d646285d..5d205ae4 100644 --- a/backend/database/mongo.go +++ b/backend/database/mongo.go @@ -93,5 +93,14 @@ func InitMongo() error { // 赋值给全局mongo session Session = sess } + //Add Unique index for 'key' + keyIndex := mgo.Index{ + Key: []string{"key"}, + Unique: true, + } + s, c := GetCol("nodes") + defer s.Close() + c.EnsureIndex(keyIndex) + return nil } From 877289e3828e8fd5603ce0bd27208298eaae70b6 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 6 Jan 2020 12:35:17 +0800 Subject: [PATCH 05/27] fixed https://github.com/crawlab-team/crawlab/issues/423 --- backend/constants/schedule.go | 2 +- backend/main.go | 14 +++++++------- backend/model/node.go | 4 ++-- backend/model/schedule.go | 32 ++++++++++++-------------------- backend/model/task.go | 16 ++++++++-------- backend/routes/schedule.go | 8 ++++---- backend/services/schedule.go | 22 +++++++++++++++++----- backend/services/task.go | 4 ++++ 8 files changed, 55 insertions(+), 47 deletions(-) diff --git a/backend/constants/schedule.go b/backend/constants/schedule.go index c3104601..520626a9 100644 --- a/backend/constants/schedule.go +++ b/backend/constants/schedule.go @@ -1,7 +1,7 @@ package constants const ( - ScheduleStatusStop = "stop" + ScheduleStatusStop = "stopped" ScheduleStatusRunning = "running" ScheduleStatusError = "error" diff --git a/backend/main.go b/backend/main.go index 74fac484..08cdf70f 100644 --- a/backend/main.go +++ b/backend/main.go @@ -189,13 +189,13 @@ func main() { authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 // 定时任务 - authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 - authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 - authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 - authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 - authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 - authGroup.POST("/schedules/:id/stop", routes.StopSchedule) // 停止定时任务 - authGroup.POST("/schedules/:id/run", routes.RunSchedule) // 运行定时任务 + authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 + authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 + authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 + authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 + authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 + authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务 + authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务 // 统计数据 authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 // 用户 diff --git a/backend/model/node.go b/backend/model/node.go index effbfbd0..88c4ed66 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -173,8 +173,8 @@ func GetNode(id bson.ObjectId) (Node, error) { defer s.Close() if err := c.FindId(id).One(&node); err != nil { - log.Errorf("get node error: %s, id: %s", err.Error(), id.Hex()) - debug.PrintStack() + //log.Errorf("get node error: %s, id: %s", err.Error(), id.Hex()) + //debug.PrintStack() return node, err } return node, nil diff --git a/backend/model/schedule.go b/backend/model/schedule.go index c1923885..3b654b74 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -16,20 +16,17 @@ type Schedule struct { Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"` - //NodeId bson.ObjectId `json:"node_id" bson:"node_id"` - //NodeKey string `json:"node_key" bson:"node_key"` Cron string `json:"cron" bson:"cron"` EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` Param string `json:"param" bson:"param"` RunType string `json:"run_type" bson:"run_type"` NodeIds []bson.ObjectId `json:"node_ids" bson:"node_ids"` - - // 状态 - Status string `json:"status" bson:"status"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` // 前端展示 SpiderName string `json:"spider_name" bson:"spider_name"` - NodeName string `json:"node_name" bson:"node_name"` + Nodes []Node `json:"nodes" bson:"nodes"` Message string `json:"message" bson:"message"` CreateTs time.Time `json:"create_ts" bson:"create_ts"` @@ -84,20 +81,15 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) { var schs []Schedule for _, schedule := range schedules { - // TODO: 获取节点名称 - //if schedule.NodeId == bson.ObjectIdHex(constants.ObjectIdNull) { - // // 选择所有节点 - // schedule.NodeName = "All Nodes" - //} else { - // // 选择单一节点 - // node, err := GetNode(schedule.NodeId) - // if err != nil { - // schedule.Status = constants.ScheduleStatusError - // schedule.Message = constants.ScheduleStatusErrorNotFoundNode - // } else { - // schedule.NodeName = node.Name - // } - //} + // 获取节点名称 + schedule.Nodes = []Node{} + if schedule.RunType == constants.RunTypeSelectedNodes { + for _, nodeId := range schedule.NodeIds { + // 选择单一节点 + node, _ := GetNode(nodeId) + schedule.Nodes = append(schedule.Nodes, node) + } + } // 获取爬虫名称 spider, err := GetSpider(schedule.SpiderId) diff --git a/backend/model/task.go b/backend/model/task.go index 299661ed..6762bd54 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -117,18 +117,12 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas for i, task := range tasks { // 获取爬虫名称 - spider, err := task.GetSpider() - if err != nil || spider.Id.Hex() == "" { - _ = spider.Delete() - } else { + if spider, err := task.GetSpider(); err == nil { tasks[i].SpiderName = spider.DisplayName } // 获取节点名称 - node, err := task.GetNode() - if node.Id.Hex() == "" || err != nil { - _ = task.Delete() - } else { + if node, err := task.GetNode(); err == nil { tasks[i].NodeName = node.Name } } @@ -142,6 +136,8 @@ func GetTaskListTotal(filter interface{}) (int, error) { var result int result, err := c.Find(filter).Count() if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() return result, err } return result, nil @@ -168,6 +164,8 @@ func AddTask(item Task) error { item.UpdateTs = time.Now() if err := c.Insert(&item); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() return err } return nil @@ -179,6 +177,8 @@ func RemoveTask(id string) error { var result Task if err := c.FindId(id).One(&result); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() return err } diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index e54c49a3..c7ef474a 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -110,9 +110,9 @@ func DeleteSchedule(c *gin.Context) { } // 停止定时任务 -func StopSchedule(c *gin.Context) { +func DisableSchedule(c *gin.Context) { id := c.Param("id") - if err := services.Sched.Stop(bson.ObjectIdHex(id)); err != nil { + if err := services.Sched.Disable(bson.ObjectIdHex(id)); err != nil { HandleError(http.StatusInternalServerError, c, err) return } @@ -120,9 +120,9 @@ func StopSchedule(c *gin.Context) { } // 运行定时任务 -func RunSchedule(c *gin.Context) { +func EnableSchedule(c *gin.Context) { id := c.Param("id") - if err := services.Sched.Run(bson.ObjectIdHex(id)); err != nil { + if err := services.Sched.Enable(bson.ObjectIdHex(id)); err != nil { HandleError(http.StatusInternalServerError, c, err) return } diff --git a/backend/services/schedule.go b/backend/services/schedule.go index 53938aea..d737c3ac 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -53,6 +53,8 @@ func AddScheduleTask(s model.Schedule) func() { Param: s.Param, } if err := AddTask(t); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() return } if err := AssignTask(t); err != nil { @@ -137,7 +139,7 @@ func (s *Scheduler) Start() error { func (s *Scheduler) AddJob(job model.Schedule) error { spec := job.Cron - // 添加任务 + // 添加定时任务 eid, err := s.cron.AddFunc(spec, AddScheduleTask(job)) if err != nil { log.Errorf("add func task error: %s", err.Error()) @@ -147,7 +149,12 @@ func (s *Scheduler) AddJob(job model.Schedule) error { // 更新EntryID job.EntryId = eid + + // 更新状态 job.Status = constants.ScheduleStatusRunning + job.Enabled = true + + // 保存定时任务 if err := job.Save(); err != nil { log.Errorf("job save error: %s", err.Error()) debug.PrintStack() @@ -176,8 +183,8 @@ func ParserCron(spec string) error { return nil } -// 停止定时任务 -func (s *Scheduler) Stop(id bson.ObjectId) error { +// 禁用定时任务 +func (s *Scheduler) Disable(id bson.ObjectId) error { schedule, err := model.GetSchedule(id) if err != nil { return err @@ -185,17 +192,22 @@ func (s *Scheduler) Stop(id bson.ObjectId) error { if schedule.EntryId == 0 { return errors.New("entry id not found") } + + // 从cron服务中删除该任务 s.cron.Remove(schedule.EntryId) + // 更新状态 schedule.Status = constants.ScheduleStatusStop + schedule.Enabled = false + if err = schedule.Save(); err != nil { return err } return nil } -// 运行任务 -func (s *Scheduler) Run(id bson.ObjectId) error { +// 启用定时任务 +func (s *Scheduler) Enable(id bson.ObjectId) error { schedule, err := model.GetSchedule(id) if err != nil { return err diff --git a/backend/services/task.go b/backend/services/task.go index c0aa8fc8..7da6b022 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -627,11 +627,15 @@ func AddTask(t model.Task) error { // 将任务存入数据库 if err := model.AddTask(t); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() return err } // 加入任务队列 if err := AssignTask(t); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() return err } From fafdccecf7f5a6d2b396421cacfef062d5ac1fed Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 6 Jan 2020 12:35:56 +0800 Subject: [PATCH 06/27] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E4=BC=98=E5=8C=96:=20=E5=8A=A0=E5=85=A5=E5=90=AF=E7=94=A8/?= =?UTF-8?q?=E7=A6=81=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/i18n/zh.js | 9 ++ frontend/src/store/modules/schedule.js | 15 ++- frontend/src/views/schedule/ScheduleList.vue | 109 +++++++++---------- frontend/src/views/task/TaskList.vue | 6 +- 4 files changed, 72 insertions(+), 67 deletions(-) diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 852e79be..ecb0bd82 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -223,6 +223,8 @@ export default { 'error': '错误', 'Not Found Node': '节点配置错误', 'Not Found Spider': '爬虫配置错误', + '[minute] [hour] [day] [month] [day of week]': '[分] [时] [天] [月] [星期几]', + 'Enable/Disable': '启用/禁用', // 网站 'Site': '网站', @@ -313,6 +315,13 @@ export default { 'Installing language failed': '安装语言失败', 'You have successfully installed a language: ': '您已成功安装语言: ', 'The language installation is unsuccessful: ': '安装语言失败: ', + 'Enabling the schedule successful': '启用定时任务成功', + 'Disabling the schedule successful': '禁用定时任务成功', + 'Enabling the schedule unsuccessful': '启用定时任务失败', + 'Disabling the schedule unsuccessful': '禁用定时任务失败', + 'The schedule has been removed': '已删除定时任务', + 'The schedule has been added': '已添加定时任务', + 'The schedule has been saved': '已保存定时任务', // 登录 'Sign in': '登录', diff --git a/frontend/src/store/modules/schedule.js b/frontend/src/store/modules/schedule.js index e71d5f03..9fde4e84 100644 --- a/frontend/src/store/modules/schedule.js +++ b/frontend/src/store/modules/schedule.js @@ -21,7 +21,12 @@ const actions = { getScheduleList ({ state, commit }) { request.get('/schedules') .then(response => { - commit('SET_SCHEDULE_LIST', response.data.data) + commit('SET_SCHEDULE_LIST', response.data.data.map(d => { + const arr = d.cron.split(' ') + arr.splice(0, 1) + d.cron = arr.join(' ') + return d + })) }) }, addSchedule ({ state }) { @@ -33,11 +38,11 @@ const actions = { removeSchedule ({ state }, id) { request.delete(`/schedules/${id}`) }, - stopSchedule ({ state, dispatch }, id) { - return request.post(`/schedules/${id}/stop`) + enableSchedule ({ state, dispatch }, id) { + return request.post(`/schedules/${id}/enable`) }, - runSchedule ({ state, dispatch }, id) { - return request.post(`/schedules/${id}/run`) + disableSchedule ({ state, dispatch }, id) { + return request.post(`/schedules/${id}/disable`) } } diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 0cfbb6d2..ab9e7636 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -67,8 +67,10 @@ - + @@ -137,13 +139,28 @@ - + + + + + + + - + @@ -41,11 +52,13 @@ import { mapGetters } from 'vuex' import Breadcrumb from '@/components/Breadcrumb' import Hamburger from '@/components/Hamburger' +import GithubButton from 'vue-github-button' export default { components: { Breadcrumb, - Hamburger + Hamburger, + GithubButton }, computed: { ...mapGetters([ @@ -122,6 +135,12 @@ export default { } } + .github { + height: 50px; + margin-right: 35px; + margin-top: -10px; + } + .right { float: right } diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 3e7bb8a3..b885732c 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -3932,6 +3932,11 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" +github-buttons@^2.3.0: + version "2.6.0" + resolved "https://registry.npm.taobao.org/github-buttons/download/github-buttons-2.6.0.tgz#fa3e031451cee7ba05c3254fa67c73fe783104dc" + integrity sha1-+j4DFFHO57oFwyVPpnxz/ngxBNw= + glob-base@^0.3.0: version "0.3.0" resolved "http://registry.npm.taobao.org/glob-base/download/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4" @@ -8587,6 +8592,13 @@ vue-eslint-parser@^5.0.0: esquery "^1.0.1" lodash "^4.17.11" +vue-github-button@^1.1.2: + version "1.1.2" + resolved "https://registry.npm.taobao.org/vue-github-button/download/vue-github-button-1.1.2.tgz#318518c3a31d0fbd278ebcc80fbc5f88d68836e6" + integrity sha1-MYUYw6MdD70njrzID7xfiNaINuY= + dependencies: + github-buttons "^2.3.0" + vue-hot-reload-api@^2.3.0: version "2.3.2" resolved "http://registry.npm.taobao.org/vue-hot-reload-api/download/vue-hot-reload-api-2.3.2.tgz#1fcc1495effe08a790909b46bf7b5c4cfeb6f21b" From bc616cdb18f582b7eecf6a1cf801ce97edbb9b9f Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 6 Jan 2020 21:12:41 +0800 Subject: [PATCH 13/27] updated README --- README-zh.md | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README-zh.md b/README-zh.md index 7848db5d..691bda88 100644 --- a/README-zh.md +++ b/README-zh.md @@ -19,8 +19,8 @@ ## 安装 三种方式: -1. [Docker](https://tikazyq.github.io/crawlab-docs/Installation/Docker.html)（推荐） -2. [直接部署](https://tikazyq.github.io/crawlab-docs/Installation/Direct.html)（了解内核） +1. [Docker](http://docs.crawlab.cn/Installation/Docker.html)（推荐） +2. [直接部署](http://docs.crawlab.cn/Installation/Direct.html)（了解内核） 3. [Kubernetes](https://juejin.im/post/5e0a02d851882549884c27ad) (多节点部署) ### 要求（Docker） diff --git a/README.md b/README.md index 6677f4a2..2b07ef1c 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ Golang-based distributed web crawler management platform, supporting various lan ## Installation Two methods: -1. [Docker](https://tikazyq.github.io/crawlab-docs/Installation/Docker.html) (Recommended) -2. [Direct Deploy](https://tikazyq.github.io/crawlab-docs/Installation/Direct.html) (Check Internal Kernel) +1. [Docker](http://docs.crawlab.cn/Installation/Docker.html) (Recommended) +2. [Direct Deploy](http://docs.crawlab.cn/Installation/Direct.html) (Check Internal Kernel) 3. [Kubernetes](https://juejin.im/post/5e0a02d851882549884c27ad) (Multi-Node Deployment) ### Pre-requisite (Docker) From fab288034dcf6729f4eeb4e574d1f40228cef084 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 6 Jan 2020 21:33:18 +0800 Subject: [PATCH 14/27] updated badges --- README-zh.md | 15 ++++++++------- README.md | 15 ++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/README-zh.md b/README-zh.md index 691bda88..f2d482c2 100644 --- a/README-zh.md +++ b/README-zh.md @@ -1,12 +1,13 @@ # Crawlab -![](http://114.67.75.98:8082/buildStatus/icon?job=crawlab%2Fmaster) -![](https://img.shields.io/github/release/crawlab-team/crawlab.svg) -![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) -![](https://img.shields.io/github/issues/crawlab-team/crawlab.svg) -![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg) -![](https://img.shields.io/docker/pulls/tikazyq/crawlab) -![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) +![](http://jenkins.crawlab.cn/buildStatus/icon?job=crawlab%2Fmaster) +![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) +![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg?logo=github) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?logo=github&logoColor=red) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?logo=github&logoColor=cyan) +![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg?logo=github) +![](https://img.shields.io/docker/pulls/tikazyq/crawlab?logo=docker) +![](https://img.shields.io/github/license/crawlab-team/crawlab.svg?logo=FreeBSD&logoColor=yellow) 中文 | [English](https://github.com/crawlab-team/crawlab) diff --git a/README.md b/README.md index 2b07ef1c..aed9d828 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ # Crawlab -![](http://114.67.75.98:8082/buildStatus/icon?job=crawlab%2Fmaster) -![](https://img.shields.io/github/release/crawlab-team/crawlab.svg) -![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) -![](https://img.shields.io/github/issues/crawlab-team/crawlab.svg) -![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg) -![](https://img.shields.io/docker/pulls/tikazyq/crawlab) -![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) +![](http://jenkins.crawlab.cn/buildStatus/icon?job=crawlab%2Fmaster) +![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) +![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg?logo=github) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?logo=github&logoColor=red) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?logo=github&logoColor=cyan) +![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg?logo=github) +![](https://img.shields.io/docker/pulls/tikazyq/crawlab?logo=docker) +![](https://img.shields.io/github/license/crawlab-team/crawlab.svg?logo=FreeBSD&logoColor=yellow) [中文](https://github.com/crawlab-team/crawlab/blob/master/README-zh.md) | English From ee0fd649cf0026efbf78f9ed9c6888dad3f8bd4d Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 6 Jan 2020 21:40:11 +0800 Subject: [PATCH 15/27] updated badges --- README-zh.md | 9 ++++----- README.md | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README-zh.md b/README-zh.md index f2d482c2..0dddf0c3 100644 --- a/README-zh.md +++ b/README-zh.md @@ -2,12 +2,11 @@ ![](http://jenkins.crawlab.cn/buildStatus/icon?job=crawlab%2Fmaster) ![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) -![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg?logo=github) -![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?logo=github&logoColor=red) -![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?logo=github&logoColor=cyan) -![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg?logo=github) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab?logo=docker) -![](https://img.shields.io/github/license/crawlab-team/crawlab.svg?logo=FreeBSD&logoColor=yellow) +![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?color=red) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?color=cyan) +![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) 中文 | [English](https://github.com/crawlab-team/crawlab) diff --git a/README.md b/README.md index aed9d828..a84ea552 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,11 @@ ![](http://jenkins.crawlab.cn/buildStatus/icon?job=crawlab%2Fmaster) ![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) -![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg?logo=github) -![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?logo=github&logoColor=red) -![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?logo=github&logoColor=cyan) -![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg?logo=github) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab?logo=docker) -![](https://img.shields.io/github/license/crawlab-team/crawlab.svg?logo=FreeBSD&logoColor=yellow) +![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?color=red) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?color=cyan) +![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) [中文](https://github.com/crawlab-team/crawlab/blob/master/README-zh.md) | English From 3cfe5c8668066653bd77a82b7956006ad0444fa7 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 6 Jan 2020 21:41:53 +0800 Subject: [PATCH 16/27] updated badges --- README-zh.md | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README-zh.md b/README-zh.md index 0dddf0c3..12b518b5 100644 --- a/README-zh.md +++ b/README-zh.md @@ -4,8 +4,8 @@ ![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab?logo=docker) ![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) -![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?color=red) -![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?color=cyan) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?label=bugs&color=red) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?label=enhancements&color=cyan) ![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) 中文 | [English](https://github.com/crawlab-team/crawlab) diff --git a/README.md b/README.md index a84ea552..236e950f 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ ![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab?logo=docker) ![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) -![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?color=red) -![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?color=cyan) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?label=bugs&color=red) +![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?label=enhancements&color=cyan) ![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) [中文](https://github.com/crawlab-team/crawlab/blob/master/README-zh.md) | English From fa4e1af1b070d2e0f47245669795717977f8f293 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 6 Jan 2020 21:43:16 +0800 Subject: [PATCH 17/27] updated badges --- README-zh.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README-zh.md b/README-zh.md index 12b518b5..944019e1 100644 --- a/README-zh.md +++ b/README-zh.md @@ -2,7 +2,7 @@ ![](http://jenkins.crawlab.cn/buildStatus/icon?job=crawlab%2Fmaster) ![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) -![](https://img.shields.io/docker/pulls/tikazyq/crawlab?logo=docker) +![](https://img.shields.io/docker/pulls/tikazyq/crawlab?label=pulls&logo=docker) ![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) ![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?label=bugs&color=red) ![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?label=enhancements&color=cyan) diff --git a/README.md b/README.md index 236e950f..efd21cb5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![](http://jenkins.crawlab.cn/buildStatus/icon?job=crawlab%2Fmaster) ![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) -![](https://img.shields.io/docker/pulls/tikazyq/crawlab?logo=docker) +![](https://img.shields.io/docker/pulls/tikazyq/crawlab?label=pulls&logo=docker) ![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) ![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?label=bugs&color=red) ![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?label=enhancements&color=cyan) From aed16d3edbc759d5e119bd223967580f42889d45 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 6 Jan 2020 21:49:58 +0800 Subject: [PATCH 18/27] updated badges --- README-zh.md | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README-zh.md b/README-zh.md index 944019e1..9057fcc3 100644 --- a/README-zh.md +++ b/README-zh.md @@ -1,8 +1,8 @@ # Crawlab -![](http://jenkins.crawlab.cn/buildStatus/icon?job=crawlab%2Fmaster) -![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) +![](https://img.shields.io/docker/cloud/build/tikazyq/crawlab.svg?label=build&logo=docker) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab?label=pulls&logo=docker) +![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) ![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) ![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?label=bugs&color=red) ![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?label=enhancements&color=cyan) diff --git a/README.md b/README.md index efd21cb5..075a80b5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Crawlab -![](http://jenkins.crawlab.cn/buildStatus/icon?job=crawlab%2Fmaster) -![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) +![](https://img.shields.io/docker/cloud/build/tikazyq/crawlab.svg?label=build&logo=docker) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab?label=pulls&logo=docker) +![](https://img.shields.io/github/release/crawlab-team/crawlab.svg?logo=github) ![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) ![](https://img.shields.io/github/issues/crawlab-team/crawlab/bug.svg?label=bugs&color=red) ![](https://img.shields.io/github/issues/crawlab-team/crawlab/enhancement.svg?label=enhancements&color=cyan) From 931e9a1d214fac5e58349e03e871aaf5d2d97141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 7 Jan 2020 00:35:56 +0800 Subject: [PATCH 19/27] =?UTF-8?q?fix=20=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 25 +++++++++++++------------ frontend/.env.development | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 5ada78f6..0ac6b5a1 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -1,25 +1,25 @@ api: address: "localhost:8000" mongo: - host: localhost + host: "192.168.235.26" port: 27017 - db: crawlab_test - username: "" - password: "" + db: crawlab_local + username: "root" + password: "example" authSource: "admin" redis: - address: localhost - password: "" + address: 127.0.0.1 + password: database: 1 port: 6379 log: level: info - path: "/var/logs/crawlab" + path: "./logs/crawlab" isDeletePeriodically: "N" deleteFrequency: "@hourly" server: host: 0.0.0.0 - port: 8000 + port: 8001 master: "Y" secret: "crawlab" register: @@ -27,11 +27,12 @@ server: type: "mac" ip: "" spider: - path: "/app/spiders" + path: "./spiders" task: workers: 4 other: - tmppath: "/tmp" -version: 0.4.2 + tmppath: "./tmp" +version: 0.4.1 setting: - allowRegister: "N" \ No newline at end of file + allowRegister: "N" + diff --git a/frontend/.env.development b/frontend/.env.development index 779d28e4..80220bf8 100644 --- a/frontend/.env.development +++ b/frontend/.env.development @@ -1,2 +1,2 @@ NODE_ENV='development' -VUE_APP_BASE_URL=http://localhost:8000 +VUE_APP_BASE_URL=http://localhost:8001 From e2eb7f6cf16c9f2a60b5bb360ded1191356546a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 7 Jan 2020 00:36:56 +0800 Subject: [PATCH 20/27] =?UTF-8?q?fix=20=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 0ac6b5a1..17dc9daf 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -35,4 +35,3 @@ other: version: 0.4.1 setting: allowRegister: "N" - From 515f3b36950ce02a0c1838bc9df5dd68e32ddb52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 7 Jan 2020 01:32:11 +0800 Subject: [PATCH 21/27] =?UTF-8?q?fix=20=E5=89=8D=E7=AB=AF=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 2 +- backend/routes/system.go | 1 + backend/services/system.go | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 9fb0dee5..6f8652f8 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -34,7 +34,7 @@ spider: task: workers: 4 other: - tmppath: "/tmp" + tmppath: "./tmp" version: 0.4.3 setting: allowRegister: "N" diff --git a/backend/routes/system.go b/backend/routes/system.go index b4e130a9..8c443d2a 100644 --- a/backend/routes/system.go +++ b/backend/routes/system.go @@ -259,6 +259,7 @@ func GetDepJson(c *gin.Context) { _dep, err := services.FetchPythonDepInfo(depName) if err != nil { HandleError(http.StatusInternalServerError, c, err) + return } dep = _dep } else { diff --git a/backend/services/system.go b/backend/services/system.go index 12b8744c..6181afee 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -251,6 +251,9 @@ func FetchPythonDepInfo(depName string) (entity.Dependency, error) { return entity.Dependency{}, err } var data PythonDepJsonData + if res.Response().StatusCode == 404 { + return entity.Dependency{}, errors.New("get depName from [https://pypi.org] error: 404") + } if err := res.ToJSON(&data); err != nil { log.Errorf(err.Error()) debug.PrintStack() From 109f1f4fdb07a048599be968c15e10150925c8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 7 Jan 2020 01:32:22 +0800 Subject: [PATCH 22/27] =?UTF-8?q?fix=20=E5=89=8D=E7=AB=AF=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/Node/NodeInstallation.vue | 14 ++++++++------ frontend/src/views/layout/components/Navbar.vue | 1 + frontend/src/views/node/NodeDetail.vue | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/Node/NodeInstallation.vue b/frontend/src/components/Node/NodeInstallation.vue index 3924cd76..1ec9894a 100644 --- a/frontend/src/components/Node/NodeInstallation.vue +++ b/frontend/src/components/Node/NodeInstallation.vue @@ -2,7 +2,7 @@
- + > - { - const res = await this.$request.get(`/system/deps/${this.activeLang.executable_name}/${dep.name}/json`) - dep.version = res.data.data.version - dep.description = res.data.data.description + const resp = await this.$request.get(`/system/deps/${this.activeLang.executable_name}/${dep.name}/json`) + if (resp) { + dep.version = resp.data.data.version + dep.description = resp.data.data.description + } }) } }, diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index 882fbfa7..caabc338 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -44,6 +44,7 @@ :aria-label="$t('Star crawlab-team/crawlab on GitHub')"> Star +
diff --git a/frontend/src/views/node/NodeDetail.vue b/frontend/src/views/node/NodeDetail.vue index dca3d818..4dab386b 100644 --- a/frontend/src/views/node/NodeDetail.vue +++ b/frontend/src/views/node/NodeDetail.vue @@ -3,7 +3,7 @@
- +
From e8a119d00db496fb8eb055ab64c439030f174811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 7 Jan 2020 01:54:49 +0800 Subject: [PATCH 23/27] =?UTF-8?q?fix=20=E6=8E=A7=E5=88=B6=E5=8F=B0?= =?UTF-8?q?=E5=A4=A7=E9=87=8F=E5=91=8A=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/InfoView/SpiderInfoView.vue | 6 +----- frontend/src/components/Stats/SpiderStats.vue | 8 ++++---- frontend/src/i18n/zh.js | 1 + frontend/src/store/modules/schedule.js | 14 ++++++++------ 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 74e48d08..93fa9c12 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -51,7 +51,7 @@ - {{$t('Run')}} @@ -69,10 +69,6 @@ {{$t('Upload')}} - - {{$t('Run')}} - {{$t('Save')}} diff --git a/frontend/src/components/Stats/SpiderStats.vue b/frontend/src/components/Stats/SpiderStats.vue index f239f823..f7993e2b 100644 --- a/frontend/src/components/Stats/SpiderStats.vue +++ b/frontend/src/components/Stats/SpiderStats.vue @@ -3,19 +3,19 @@
- - - - diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index b15efd50..e739b979 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -225,6 +225,7 @@ export default { 'Not Found Spider': '爬虫配置错误', '[minute] [hour] [day] [month] [day of week]': '[分] [时] [天] [月] [星期几]', 'Enable/Disable': '启用/禁用', + 'Cron': 'Cron', // 网站 'Site': '网站', diff --git a/frontend/src/store/modules/schedule.js b/frontend/src/store/modules/schedule.js index 9fde4e84..9e60169c 100644 --- a/frontend/src/store/modules/schedule.js +++ b/frontend/src/store/modules/schedule.js @@ -21,12 +21,14 @@ const actions = { getScheduleList ({ state, commit }) { request.get('/schedules') .then(response => { - commit('SET_SCHEDULE_LIST', response.data.data.map(d => { - const arr = d.cron.split(' ') - arr.splice(0, 1) - d.cron = arr.join(' ') - return d - })) + if (response.data.data) { + commit('SET_SCHEDULE_LIST', response.data.data.map(d => { + const arr = d.cron.split(' ') + arr.splice(0, 1) + d.cron = arr.join(' ') + return d + })) + } }) }, addSchedule ({ state }) { From 71ca84b5a910a541f0142c3771a97297914c0608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 7 Jan 2020 02:07:52 +0800 Subject: [PATCH 24/27] =?UTF-8?q?=E8=BF=98=E5=8E=9Fconfig.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 6f8652f8..a51a34c9 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -1,25 +1,25 @@ api: address: "localhost:8000" mongo: - host: "192.168.235.26" + host: localhost port: 27017 - db: crawlab_local - username: "root" - password: "example" + db: crawlab_test + username: "" + password: "" authSource: "admin" redis: - address: 127.0.0.1 - password: + address: localhost + password: "" database: 1 port: 6379 log: level: info - path: "./logs/crawlab" + path: "/var/logs/crawlab" isDeletePeriodically: "N" deleteFrequency: "@hourly" server: host: 0.0.0.0 - port: 8001 + port: 8000 master: "Y" secret: "crawlab" register: @@ -30,11 +30,11 @@ server: python: "Y" node: "N" spider: - path: "./spiders" + path: "/app/spiders" task: workers: 4 other: - tmppath: "./tmp" + tmppath: "/tmp" version: 0.4.3 setting: - allowRegister: "N" + allowRegister: "N" \ No newline at end of file From afd55c1b0d10ccf3f00cbd1c30f9381a277ae35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 7 Jan 2020 02:08:41 +0800 Subject: [PATCH 25/27] =?UTF-8?q?fix=20UI=E9=94=99=E8=AF=AF=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/layout/components/Navbar.vue | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index caabc338..f47380c8 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -27,14 +27,13 @@ - +
{{$t('Documentation')}} - - - +
+
Star - - +
@@ -130,7 +128,8 @@ export default { .documentation { margin-right: 35px; - + color: #606266; + font-size: 14px; .span { margin-left: 5px; } From 1f63f6fb307f665d0568b18dfc7730dfa5ff91fc Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 7 Jan 2020 12:42:01 +0800 Subject: [PATCH 26/27] fixed api url issue --- frontend/.env.development | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/.env.development b/frontend/.env.development index 80220bf8..779d28e4 100644 --- a/frontend/.env.development +++ b/frontend/.env.development @@ -1,2 +1,2 @@ NODE_ENV='development' -VUE_APP_BASE_URL=http://localhost:8001 +VUE_APP_BASE_URL=http://localhost:8000 From e0659809a2f85ea5ab20ce30865ddccfe8ebc013 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 7 Jan 2020 13:34:57 +0800 Subject: [PATCH 27/27] updated docker-compose.yml --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5c059f95..fcab07f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,6 @@ services: depends_on: - mongo - redis - volumes: - - "/Users/marvzhang/projects/crawlab-team/crawlab/docker_init.sh:/app/docker_init.sh" worker: image: tikazyq/crawlab:latest container_name: worker