diff --git a/backend/entity/spider.go b/backend/entity/spider.go index 6f8fbee1..616d3bbf 100644 --- a/backend/entity/spider.go +++ b/backend/entity/spider.go @@ -6,7 +6,12 @@ type SpiderType struct { } type ScrapySettingParam struct { - Key string - Value interface{} - Type string + Key string `json:"key"` + Value interface{} `json:"value"` + Type string `json:"type"` +} + +type ScrapyItem struct { + Name string `json:"name"` + Fields []string `json:"fields"` } diff --git a/backend/main.go b/backend/main.go index 6c00c797..5edf4e6c 100644 --- a/backend/main.go +++ b/backend/main.go @@ -155,31 +155,35 @@ func main() { } // 爬虫 { - authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 - authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 - authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 - authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 - authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 - authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 - authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） - authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 - authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 - authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 - authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 - authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 - authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 - authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 - authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 - authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 - authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 - authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 - authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 - authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 - authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫 - authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置 - authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 - authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 - authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 + authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 + authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 + authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 + authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 + authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 + authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 + authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） + authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 + authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 + authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 + authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 + authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 + authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 + authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 + authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 + authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 + authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 + authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 + authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 + authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫 + authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置 + authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 + authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items + authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items + authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines + authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines + authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 + authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 } // 可配置爬虫 { diff --git a/backend/routes/spider.go b/backend/routes/spider.go index dc84f462..d53d9715 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -187,19 +187,33 @@ func PutSpider(c *gin.Context) { // 将FileId置空 spider.FileId = bson.ObjectIdHex(constants.ObjectIdNull) - // 创建爬虫目录 + // 爬虫目录 spiderDir := filepath.Join(viper.GetString("spider.path"), spider.Name) + + // 赋值到爬虫实例 + spider.Src = spiderDir + + // 移除已有爬虫目录 if utils.Exists(spiderDir) { if err := os.RemoveAll(spiderDir); err != nil { HandleError(http.StatusInternalServerError, c, err) return } } + + // 生成爬虫目录 if err := os.MkdirAll(spiderDir, 0777); err != nil { HandleError(http.StatusInternalServerError, c, err) return } - spider.Src = spiderDir + + // 如果为 Scrapy 项目，生成 Scrapy 项目 + if spider.IsScrapy { + if err := services.CreateScrapyProject(spider); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } // 添加爬虫到数据库 if err := spider.Add(); err != nil { @@ -960,8 +974,9 @@ func GetSpiderScrapySpiders(c *gin.Context) { func PutSpiderScrapySpiders(c *gin.Context) { type ReqBody struct { - Name string `json:"name"` - Domain string `json:"domain"` + Name string `json:"name"` + Domain string `json:"domain"` + Template string `json:"template"` } id := c.Param("id") @@ -983,7 +998,7 @@ func PutSpiderScrapySpiders(c *gin.Context) { return } - if err := services.CreateScrapySpider(spider, reqBody.Name, reqBody.Domain); err != nil { + if err := services.CreateScrapySpider(spider, reqBody.Name, reqBody.Domain, reqBody.Template); err != nil { HandleError(http.StatusInternalServerError, c, err) return } @@ -1052,6 +1067,124 @@ func PostSpiderScrapySettings(c *gin.Context) { }) } +func GetSpiderScrapyItems(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + data, err := services.GetScrapyItems(spider) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: data, + }) +} + +func PostSpiderScrapyItems(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + var reqData []entity.ScrapyItem + if err := c.ShouldBindJSON(&reqData); err != nil { + HandleErrorF(http.StatusBadRequest, c, "invalid request") + return + } + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + if err := services.SaveScrapyItems(spider, reqData); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func GetSpiderScrapyPipelines(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + data, err := services.GetScrapyPipelines(spider) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: data, + }) +} + +func GetSpiderScrapySpiderFilepath(c *gin.Context) { + id := c.Param("id") + + spiderName := c.Query("spider_name") + if spiderName == "" { + HandleErrorF(http.StatusBadRequest, c, "spider_name is empty") + return + } + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + data, err := services.GetScrapySpiderFilepath(spider, spiderName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: data, + }) +} + func PostSpiderSyncGit(c *gin.Context) { id := c.Param("id") diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go index 374df4ee..5b91c501 100644 --- a/backend/services/scrapy.go +++ b/backend/services/scrapy.go @@ -135,16 +135,147 @@ func SaveScrapySettings(s model.Spider, settingsData []entity.ScrapySettingParam return } -func CreateScrapySpider(s model.Spider, name string, domain string) (err error) { +func GetScrapyItems(s model.Spider) (res []map[string]interface{}, err error) { var stdout bytes.Buffer var stderr bytes.Buffer - cmd := exec.Command("scrapy", "genspider", name, domain) + cmd := exec.Command("crawlab", "items") cmd.Dir = s.Src cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Run(); err != nil { log.Errorf(err.Error()) + log.Errorf(stderr.String()) + debug.PrintStack() + return res, err + } + + if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return res, err + } + + return res, nil +} + +func SaveScrapyItems(s model.Spider, itemsData []entity.ScrapyItem) (err error) { + // 读取 scrapy.cfg + cfg, err := goconfig.LoadConfigFile(path.Join(s.Src, "scrapy.cfg")) + if err != nil { + return + } + modName, err := cfg.GetValue("settings", "default") + if err != nil { + return + } + + // 定位到 settings.py 文件 + arr := strings.Split(modName, ".") + dirName := arr[0] + fileName := "items" + filePath := fmt.Sprintf("%s/%s/%s.py", s.Src, dirName, fileName) + + // 生成文件内容 + content := "" + content += "import scrapy\n" + content += "\n\n" + for _, item := range itemsData { + content += fmt.Sprintf("class %s(scrapy.Item):\n", item.Name) + for _, field := range item.Fields { + content += fmt.Sprintf(" %s = scrapy.Field()\n", field) + } + content += "\n\n" + } + + // 写到 settings.py + if err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm); err != nil { + return err + } + + // 同步到GridFS + if err := UploadSpiderToGridFsFromMaster(s); err != nil { + return err + } + + return +} + +func GetScrapyPipelines(s model.Spider) (res []string, err error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("crawlab", "pipelines") + cmd.Dir = s.Src + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf(stderr.String()) + debug.PrintStack() + return res, err + } + + if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return res, err + } + + return res, nil +} + +func GetScrapySpiderFilepath(s model.Spider, spiderName string) (res string, err error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("crawlab", "find_spider_filepath", "-n", spiderName) + cmd.Dir = s.Src + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf(stderr.String()) + debug.PrintStack() + return res, err + } + + res = strings.Replace(stdout.String(), "\n", "", 1) + + return res, nil +} + +func CreateScrapySpider(s model.Spider, name string, domain string, template string) (err error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("scrapy", "genspider", name, domain, "-t", template) + cmd.Dir = s.Src + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf("stdout: " + stdout.String()) + log.Errorf("stderr: " + stderr.String()) + debug.PrintStack() + return err + } + + return +} + +func CreateScrapyProject(s model.Spider) (err error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("scrapy", "startproject", s.Name, s.Src) + cmd.Dir = s.Src + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf("stdout: " + stdout.String()) + log.Errorf("stderr: " + stderr.String()) debug.PrintStack() return err } diff --git a/frontend/src/components/File/FileList.vue b/frontend/src/components/File/FileList.vue index d4f6106d..935dc0fe 100644 --- a/frontend/src/components/File/FileList.vue +++ b/frontend/src/components/File/FileList.vue @@ -409,6 +409,30 @@ export default { this.isShowDelete = false this.showFile = false this.$st.sendEv('爬虫详情', '文件', '删除') + }, + clickSpider (filepath) { + const node = this.$refs['tree'].getNode(filepath) + const data = node.data + this.onFileClick(data) + node.parent.expanded = true + node.parent.parent.expanded = true + }, + clickPipeline () { + const filename = 'pipelines.py' + for (let i = 0; i < this.computedFileTree.length; i++) { + const dataLv1 = this.computedFileTree[i] + const nodeLv1 = this.$refs['tree'].getNode(dataLv1.path) + if (dataLv1.is_dir) { + for (let j = 0; j < dataLv1.children.length; j++) { + const dataLv2 = dataLv1.children[j] + if (dataLv2.path.match(filename)) { + this.onFileClick(dataLv2) + nodeLv1.expanded = true + return + } + } + } + } } }, async created () { diff --git a/frontend/src/components/Scrapy/SpiderScrapy.vue b/frontend/src/components/Scrapy/SpiderScrapy.vue index b183f72a..784b2c44 100644 --- a/frontend/src/components/Scrapy/SpiderScrapy.vue +++ b/frontend/src/components/Scrapy/SpiderScrapy.vue @@ -1,5 +1,6 @@ + + + + + + + + + + + -
-

{{$t('Scrapy Spiders')}}

-
- - {{$t('Add Spider')}} - -
- -
-
-

{{$t('Settings')}}

-
- - {{$t('Add')}} - - - {{$t('Save')}} - -
- - - - - - - - - - - - - - -
+ type="primary" + size="small" + icon="el-icon-plus" + @click="onSettingsAdd" + > + {{$t('Add Variable')}} + + + {{$t('Save')}} + + + + + + + + + + + + + + + + + + + + + + +
+
+ + {{$t('Add Spider')}} + +
+ +
+ + + + + +
+
+ + {{$t('Add Item')}} + + + {{$t('Save')}} + +
+ + + + + + +
+ + + + + +
+ +
+ + + @@ -247,7 +374,9 @@ export default { computed: { ...mapState('spider', [ 'spiderForm', - 'spiderScrapySettings' + 'spiderScrapySettings', + 'spiderScrapyItems', + 'spiderScrapyPipelines' ]), activeParamData () { if (this.activeParam.type === 'array') { @@ -273,9 +402,12 @@ export default { isAddSpiderVisible: false, addSpiderForm: { name: '', - domain: '' + domain: '', + template: 'basic' }, - isAddSpiderLoading: false + isAddSpiderLoading: false, + activeTabName: 'settings', + loadingDict: {} } }, methods: { @@ -285,7 +417,7 @@ export default { onCloseDialog () { this.dialogVisible = false }, - onConfirm () { + onSettingsConfirm () { if (this.activeParam.type === 'array') { this.activeParam.value = this.activeParamData.map(d => d.value) } else if (this.activeParam.type === 'object') { @@ -297,22 +429,22 @@ export default { } this.$set(this.spiderScrapySettings, this.activeParamIndex, JSON.parse(JSON.stringify(this.activeParam))) this.dialogVisible = false - this.$st('爬虫详情', 'Scrapy 设置', '确认编辑参数') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '确认编辑参数') }, - onEditParam (row, index) { + onSettingsEditParam (row, index) { this.activeParam = JSON.parse(JSON.stringify(row)) this.activeParamIndex = index this.onOpenDialog() - this.$st('爬虫详情', 'Scrapy 设置', '点击编辑参数') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '点击编辑参数') }, - async onSave () { + async onSettingsSave () { const res = await this.$store.dispatch('spider/saveSpiderScrapySettings', this.$route.params.id) if (!res.data.error) { this.$message.success(this.$t('Saved successfully')) } - this.$st('爬虫详情', 'Scrapy 设置', '保存设置') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '保存设置') }, - onAdd () { + onSettingsAdd () { const data = JSON.parse(JSON.stringify(this.spiderScrapySettings)) data.push({ key: '', @@ -320,15 +452,15 @@ export default { type: 'string' }) this.$store.commit('spider/SET_SPIDER_SCRAPY_SETTINGS', data) - this.$st('爬虫详情', 'Scrapy 设置', '添加参数') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加参数') }, - onRemove (index) { + onSettingsRemove (index) { const data = JSON.parse(JSON.stringify(this.spiderScrapySettings)) data.splice(index, 1) this.$store.commit('spider/SET_SPIDER_SCRAPY_SETTINGS', data) - this.$st('爬虫详情', 'Scrapy 设置', '删除参数') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '删除参数') }, - onActiveParamAdd () { + onSettingsActiveParamAdd () { if (this.activeParam.type === 'array') { this.activeParam.value.push('') } else if (this.activeParam.type === 'object') { @@ -337,9 +469,9 @@ export default { } this.$set(this.activeParam.value, '', 999) } - this.$st('爬虫详情', 'Scrapy 设置', '添加参数中参数') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加参数中参数') }, - onActiveParamRemove (index) { + onSettingsActiveParamRemove (index) { if (this.activeParam.type === 'array') { this.activeParam.value.splice(index, 1) } else if (this.activeParam.type === 'object') { @@ -348,7 +480,7 @@ export default { delete value[key] this.$set(this.activeParam, 'value', value) } - this.$st('爬虫详情', 'Scrapy 设置', '删除参数中参数') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '删除参数中参数') }, settingsKeysFetchSuggestions (queryString, cb) { const data = this.$utils.scrapy.settingParamNames @@ -367,7 +499,7 @@ export default { }) cb(data) }, - onParamTypeChange (row) { + onSettingsParamTypeChange (row) { if (row.type === 'number') { row.value = Number(row.value) } @@ -387,15 +519,138 @@ export default { this.isAddSpiderLoading = false await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id) }) - this.$st('爬虫详情', 'Scrapy 设置', '确认添加爬虫') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '确认添加爬虫') }, onAddSpider () { this.addSpiderForm = { name: '', - domain: '' + domain: '', + template: 'basic' } this.isAddSpiderVisible = true - this.$st('爬虫详情', 'Scrapy 设置', '添加爬虫') + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加爬虫') + }, + getMaxItemNodeId () { + let max = 0 + this.spiderScrapyItems.forEach(d => { + if (max < d.id) max = d.id + d.children.forEach(f => { + if (max < f.id) max = f.id + }) + }) + return max + }, + onAddItem () { + const maxId = this.getMaxItemNodeId() + this.spiderScrapyItems.push({ + id: maxId + 1, + label: `Item_${+new Date()}`, + level: 1, + children: [ + { + id: maxId + 2, + level: 2, + label: `field_${+new Date()}` + } + ] + }) + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加Item') + }, + removeItem (data, ev) { + ev.stopPropagation() + for (let i = 0; i < this.spiderScrapyItems.length; i++) { + const item = this.spiderScrapyItems[i] + if (item.id === data.id) { + this.spiderScrapyItems.splice(i, 1) + break + } + } + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '删除Item') + }, + onAddItemField (data, ev) { + ev.stopPropagation() + for (let i = 0; i < this.spiderScrapyItems.length; i++) { + const item = this.spiderScrapyItems[i] + if (item.id === data.id) { + item.children.push({ + id: this.getMaxItemNodeId() + 1, + level: 2, + label: `field_${+new Date()}` + }) + break + } + } + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加Items字段') + }, + onRemoveItemField (node, data, ev) { + ev.stopPropagation() + for (let i = 0; i < this.spiderScrapyItems.length; i++) { + const item = this.spiderScrapyItems[i] + if (item.id === node.parent.data.id) { + for (let j = 0; j < item.children.length; j++) { + const field = item.children[j] + if (field.id === data.id) { + item.children.splice(j, 1) + break + } + } + break + } + } + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '删除Items字段') + }, + onItemLabelEdit (node, data, ev) { + ev.stopPropagation() + this.$set(node, 'isEdit', true) + this.$set(data, 'name', node.label) + setTimeout(() => { + this.$refs[`el-input-${data.id}`].focus() + }, 0) + }, + onItemChange (node, data, value) { + for (let i = 0; i < this.spiderScrapyItems.length; i++) { + const item = this.spiderScrapyItems[i] + if (item.id === data.id) { + item.label = value + break + } + } + }, + onItemFieldChange (node, data, value) { + for (let i = 0; i < this.spiderScrapyItems.length; i++) { + const item = this.spiderScrapyItems[i] + if (item.id === node.parent.data.id) { + for (let j = 0; j < item.children.length; j++) { + const field = item.children[j] + if (field.id === data.id) { + item.children[j].label = value + break + } + } + break + } + } + }, + async onItemsSave () { + const res = await this.$store.dispatch('spider/saveSpiderScrapyItems', this.$route.params.id) + if (!res.data.error) { + this.$message.success(this.$t('Saved successfully')) + } + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '保存Items') + }, + async onClickSpider (spiderName) { + if (this.loadingDict[spiderName]) return + this.$set(this.loadingDict, spiderName, true) + try { + const res = await this.$store.dispatch('spider/getSpiderScrapySpiderFilepath', { + id: this.$route.params.id, + spiderName + }) + this.$emit('click-spider', res.data.data) + } finally { + this.$set(this.loadingDict, spiderName, false) + } + this.$st.sendEv('爬虫详情', 'Scrapy 设置', '点击爬虫') } } } @@ -407,49 +662,16 @@ export default { color: #606266; } - .spiders { - float: left; - display: inline-block; - width: 240px; - height: 100%; - border: 1px solid #DCDFE6; - border-radius: 3px; - padding: 0 10px; + .spider-scrapy >>> .el-tabs__content { + overflow: auto; } - .spiders .title { - border-bottom: 1px solid #DCDFE6; - padding-bottom: 15px; - } - - .spiders .action-wrapper { - margin-bottom: 10px; - text-align: right; - } - - .spiders .spider-list { - list-style: none; - padding: 0; - margin: 0; - } - - .spiders .spider-list .spider-item { - padding: 10px; - cursor: pointer; - } - - .spiders .spider-list .spider-item:hover { - background: #F5F7FA; + .spider-scrapy >>> .el-tab-pane { + height: calc(100vh - 239px); } .settings { - margin-left: 20px; - border: 1px solid #DCDFE6; - float: left; - width: calc(100% - 240px - 20px); - height: 100%; - border-radius: 3px; - padding: 0 20px; + width: 100%; } .settings .title { @@ -485,4 +707,77 @@ export default { .settings >>> .top-action-wrapper .el-button { margin-left: 10px; } + + .spiders { + width: 100%; + height: auto; + overflow: auto; + } + + .spiders .action-wrapper { + text-align: right; + padding-bottom: 10px; + border-bottom: 1px solid #DCDFE6; + } + + .pipelines .list, + .spiders .list { + list-style: none; + padding: 0; + margin: 0; + } + + .pipelines .list .item, + .spiders .list .item { + font-size: 14px; + padding: 10px; + cursor: pointer; + } + + .pipelines .list .item:hover, + .spiders .list .item:hover { + background: #F5F7FA; + } + + .items { + width: 100%; + height: auto; + overflow: auto; + } + + .items >>> .action-wrapper { + text-align: right; + padding-bottom: 10px; + border-bottom: 1px solid #DCDFE6; + } + + .items >>> .custom-tree-node { + flex: 1; + display: flex; + align-items: center; + justify-content: space-between; + font-size: 14px; + padding-right: 8px; + min-height: 36px; + } + + .items >>> .el-tree > .el-tree-node { + border-bottom: 1px solid #e6e9f0; + } + + .items >>> .el-tree-node__content { + height: auto; + } + + .items >>> .custom-tree-node .label i.el-icon-edit { + visibility: hidden; + } + + .items >>> .custom-tree-node:hover .label i.el-icon-edit { + visibility: visible; + } + + .items >>> .custom-tree-node .el-input { + width: 240px; + } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 5c7e03e2..01759c8a 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -217,6 +217,9 @@ export default { 'Long Task': '长任务', 'Running Task Count': '运行中的任务数', 'Running Tasks': '运行中的任务', + 'Item Name': 'Item 名称', + 'Add Item': '添加 Item', + 'Add Variable': '添加变量', // 爬虫列表 'Name': '名称', diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index c5a84ef1..f4e5dd02 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -13,6 +13,12 @@ const state = { // spider scrapy settings spiderScrapySettings: [], + // spider scrapy items + spiderScrapyItems: [], + + // spider scrapy pipelines + spiderScrapyPipelines: [], + // node to deploy/run activeNode: {}, @@ -98,6 +104,12 @@ const mutations = { }, SET_SPIDER_SCRAPY_SETTINGS (state, value) { state.spiderScrapySettings = value + }, + SET_SPIDER_SCRAPY_ITEMS (state, value) { + state.spiderScrapyItems = value + }, + SET_SPIDER_SCRAPY_PIPELINES (state, value) { + state.spiderScrapyPipelines = value } } @@ -150,6 +162,43 @@ const actions = { async saveSpiderScrapySettings ({ state }, id) { return request.post(`/spiders/${id}/scrapy/settings`, state.spiderScrapySettings) }, + async getSpiderScrapyItems ({ state, commit }, id) { + const res = await request.get(`/spiders/${id}/scrapy/items`) + let nodeId = 0 + commit('SET_SPIDER_SCRAPY_ITEMS', res.data.data.map(d => { + d.id = nodeId++ + d.label = d.name + d.level = 1 + d.isEdit = false + d.children = d.fields.map(f => { + return { + id: nodeId++, + label: f, + level: 2, + isEdit: false + } + }) + return d + })) + }, + async saveSpiderScrapyItems ({ state }, id) { + return request.post(`/spiders/${id}/scrapy/items`, state.spiderScrapyItems.map(d => { + d.name = d.label + d.fields = d.children.map(f => f.label) + return d + })) + }, + async getSpiderScrapyPipelines ({ state, commit }, id) { + const res = await request.get(`/spiders/${id}/scrapy/pipelines`) + commit('SET_SPIDER_SCRAPY_PIPELINES', res.data.data) + }, + async saveSpiderScrapyPipelines ({ state }, id) { + return request.post(`/spiders/${id}/scrapy/pipelines`, state.spiderScrapyPipelines) + }, + async getSpiderScrapySpiderFilepath ({ state, commit }, payload) { + const { id, spiderName } = payload + return request.get(`/spiders/${id}/scrapy/spider/filepath`, { spider_name: spiderName }) + }, addSpiderScrapySpider ({ state }, payload) { const { id, form } = payload return request.put(`/spiders/${id}/scrapy/spiders`, form) diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index 72555bd7..5e380e1b 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -180,7 +180,6 @@ docker-compose up -d }, howToUpgradeHtml () { if (this.lang === 'zh') { - console.log(this.howToUpgradeHtmlZh) return this.converter.makeHtml(this.howToUpgradeHtmlZh) } else if (this.lang === 'en') { return this.converter.makeHtml(this.howToUpgradeHtmlEn) diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index a55a5212..787788d3 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -26,13 +26,18 @@ - + - + @@ -162,7 +167,8 @@ export default { } this.$utils.tour.nextStep('spider-detail', currentStep) } - } + }, + redirectType: '' } }, computed: { @@ -190,7 +196,7 @@ export default { } }, methods: { - onTabClick (tab) { + async onTabClick (tab) { if (this.activeTabName === 'analytics') { setTimeout(() => { this.$refs['spider-stats'].update() @@ -207,12 +213,11 @@ export default { }, 100) } } else if (this.activeTabName === 'scrapy-settings') { - this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id) - this.$store.dispatch('spider/getSpiderScrapySettings', this.$route.params.id) + await this.getScrapyData() } else if (this.activeTabName === 'files') { - this.$store.dispatch('spider/getFileTree') + await this.$store.dispatch('spider/getFileTree') if (this.currentPath) { - this.$store.dispatch('file/getFileContent', { path: this.currentPath }) + await this.$store.dispatch('file/getFileContent', { path: this.currentPath }) } } this.$st.sendEv('爬虫详情', '切换标签', tab.name) @@ -220,12 +225,27 @@ export default { onSpiderChange (id) { this.$router.push(`/spiders/${id}`) this.$st.sendEv('爬虫详情', '切换爬虫') + }, + async getScrapyData () { + await Promise.all([ + this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id), + this.$store.dispatch('spider/getSpiderScrapyItems', this.$route.params.id), + this.$store.dispatch('spider/getSpiderScrapySettings', this.$route.params.id), + this.$store.dispatch('spider/getSpiderScrapyPipelines', this.$route.params.id) + ]) + }, + async onClickScrapySpider (filepath) { + this.activeTabName = 'files' + await this.$store.dispatch('spider/getFileTree') + this.$refs['file-list'].clickSpider(filepath) + }, + async onClickScrapyPipeline () { + this.activeTabName = 'files' + await this.$store.dispatch('spider/getFileTree') + this.$refs['file-list'].clickPipeline() } }, async created () { - // get the list of the spiders - // this.$store.dispatch('spider/getSpiderList') - // get spider basic info await this.$store.dispatch('spider/getSpiderData', this.$route.params.id) @@ -237,12 +257,6 @@ export default { // get spider list await this.$store.dispatch('spider/getSpiderList') - - // get scrapy spider names - if (this.spiderForm.is_scrapy) { - await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id) - await this.$store.dispatch('spider/getSpiderScrapySettings', this.$route.params.id) - } }, mounted () { if (!this.$utils.tour.isFinishedTour('spider-detail')) { diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index ba6751a5..049a9b26 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -45,7 +45,12 @@ - + @@ -64,18 +69,33 @@ - - - - - - + + + + + + + + + + + + + + + + +