git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入批量删除爬虫任务

Browse Source
This commit is contained in:
marvzhang
2020-02-22 17:35:51 +08:00
parent eb167ead1e
commit 2e6b6e4e70
11 changed files with 95 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
backend/main.go
View File

@@ -185,6 +185,7 @@ func main() {
authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines
authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步
authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置
authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务
}
// 可配置爬虫
{
@@ -202,8 +203,8 @@ func main() {
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果

4
backend/model/spider.go
View File

@@ -279,6 +279,8 @@ func RemoveSpider(id bson.ObjectId) error {
var result Spider
if err := c.FindId(id).One(&result); err != nil {
log.Errorf("find spider error: %s, id:%s", err.Error(), id.Hex())
debug.PrintStack()
return err
}
@@ -291,12 +293,10 @@ func RemoveSpider(id bson.ObjectId) error {
// gf上的文件
s, gf := database.GetGridFs("files")
defer s.Close()
if result.FileId.Hex() != constants.ObjectIdNull {
if err := gf.RemoveId(result.FileId); err != nil {
log.Error("remove file error, id:" + result.FileId.Hex())
debug.PrintStack()
return err
}
}

24
backend/routes/spider.go
View File

@@ -512,6 +512,30 @@ func DeleteSelectedSpider(c *gin.Context) {
})
}
func CancelSelectedSpider(c *gin.Context) {
type ReqBody struct {
SpiderIds []string `json:"spider_ids"`
}
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleErrorF(http.StatusBadRequest, c, "invalid request")
return
}
for _, spiderId := range reqBody.SpiderIds {
if err := services.CancelSpider(spiderId); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
}
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func GetSpiderTasks(c *gin.Context) {
id := c.Param("id")

2
backend/routes/task.go
View File

@@ -183,7 +183,7 @@ func DeleteTaskByStatus(c *gin.Context) {
}
// 删除多个任务
func DeleteMultipleTask(c *gin.Context) {
func DeleteSelectedTask(c *gin.Context) {
ids := make(map[string][]string)
if err := c.ShouldBindJSON(&ids); err != nil {
HandleError(http.StatusInternalServerError, c, err)

6
backend/routes/version.go
View File

@@ -2,15 +2,17 @@ package routes
import (
"crawlab/services"
"github.com/apex/log"
"github.com/gin-gonic/gin"
"net/http"
"runtime/debug"
)
func GetLatestRelease(c *gin.Context) {
latestRelease, err := services.GetLatestRelease()
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
log.Errorf(err.Error())
debug.PrintStack()
}
c.JSON(http.StatusOK, Response{
Status: "ok",

32
backend/services/spider.go
View File

@@ -261,6 +261,38 @@ func RemoveSpider(id string) error {
return nil
}
func CancelSpider(id string) error {
// 获取该爬虫
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
return err
}
// 获取该爬虫待定或运行中的任务列表
query := bson.M{
"spider_id": spider.Id,
"status": bson.M{
"$in": []string{
constants.StatusPending,
constants.StatusRunning,
},
},
}
tasks, err := model.GetTaskList(query, 0, constants.Infinite, "-create_ts")
if err != nil {
return err
}
// 遍历任务列表，依次停止
for _, task := range tasks {
if err := CancelTask(task.Id); err != nil {
return err
}
}
return nil
}
// 启动爬虫服务
func InitSpiderService() error {
// 构造定时任务执行器

4
frontend/src/components/File/FileList.vue
View File

@@ -25,8 +25,8 @@
</el-form-item>
</el-form>
<span slot="footer" class="dialog-footer">
<el-button @click="fileDialogVisible = false">{{$t('Cancel')}}</el-button>
<el-button type="primary" @click="onAddFile">{{$t('Confirm')}}</el-button>
<el-button size="small" @click="fileDialogVisible = false">{{$t('Cancel')}}</el-button>
<el-button size="small" type="primary" @click="onAddFile">{{$t('Confirm')}}</el-button>
</span>
</el-dialog>

2
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -45,7 +45,7 @@
/>
</el-form-item>
</template>
<el-form-item :label="$t('Results Collection')" prop="col" required :inline-message="true">
<el-form-item :label="$t('Results Collection')" prop="col">
<el-input v-model="spiderForm.col" :placeholder="$t('Results Collection')"
:disabled="isView"></el-input>
</el-form-item>

2
frontend/src/i18n/zh.js
View File

@@ -560,6 +560,8 @@ docker run -d --restart always --name crawlab_worker \\
'SSH Public Key is copied to the clipboard': 'SSH 公钥已粘贴到剪切板',
'Removed successfully': '已成功删除',
'Are you sure to delete selected items?': '您是否确认删除所选项？',
'Are you sure to stop selected items?': '您是否确认停止所选项？',
'Sent signals to cancel selected tasks': '已经向所选任务发送取消任务信号',
// 其他
'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'

2
frontend/src/store/modules/task.js
View File

@@ -136,7 +136,7 @@ const actions = {
})
},
deleteTaskMultiple ({ state }, ids) {
return request.delete(`/tasks_multiple`, {
return request.delete(`/tasks`, {
ids: ids
})
},

24
frontend/src/views/spider/SpiderList.vue
View File

@@ -52,7 +52,7 @@
:disabled="spiderForm.is_scrapy"
/>
</el-form-item>
<el-form-item :label="$t('Results')" prop="col" required>
<el-form-item :label="$t('Results')" prop="col">
<el-input id="col" v-model="spiderForm.col" :placeholder="$t('Results')"/>
</el-form-item>
<el-form-item :label="$t('Upload Zip File')" label-width="120px" name="site">
@@ -1143,6 +1143,28 @@ export default {
}
this.$st.sendEv('爬虫列表', '批量删除爬虫')
})
},
async onStopSelectedSpiders () {
this.$confirm(this.$t('Are you sure to stop selected items?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel'),
type: 'warning'
}).then(async () => {
this.isStopLoading = true
try {
const res = await this.$request.post('/spiders-cancel', {
spider_ids: this.selectedSpiders.map(d => d._id)
})
if (!res.data.error) {
this.$message.success('Sent signals to cancel selected tasks')
this.$refs['table'].clearSelection()
await this.getList()
}
} finally {
this.isStopLoading = false
}
this.$st.sendEv('爬虫列表', '批量删除爬虫')
})
}
},
async created () {