加入复制爬虫

backend/routes/spider.go

@@ -259,14 +259,25 @@ func CopySpider(c *gin.Context) {
 		return
 	}
 
+	// 检查新爬虫名称是否存在
+	// 如果存在，则返回错误
+	s := model.GetSpiderByName(reqBody.Name)
+	if s.Name != "" {
+		HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("spider name '%s' already exists", reqBody.Name))
+		return
+	}
+
+	// 被复制爬虫
 	spider, err := model.GetSpider(bson.ObjectIdHex(id))
 	if err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return
 	}
 
+	// 复制爬虫
 	if err := services.CopySpider(spider, reqBody.Name); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
+		return
 	}
 
 	c.JSON(http.StatusOK, Response{

backend/services/spider.go

@@ -15,11 +15,13 @@ import (
 	"github.com/satori/go.uuid"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"runtime/debug"
+	"time"
 )
 
 type SpiderFileData struct {
@@ -293,7 +295,121 @@ func CancelSpider(id string) error {
 	return nil
 }
 
+func cloneGridFsFile(spider model.Spider, newName string) (err error) {
+	// 构造新爬虫
+	newSpider := spider
+	newSpider.Id = bson.NewObjectId()
+	newSpider.Name = newName
+	newSpider.DisplayName = newName
+	newSpider.Src = path.Join(path.Dir(spider.Src), newName)
+	newSpider.CreateTs = time.Now()
+	newSpider.UpdateTs = time.Now()
+
+	// GridFS连接实例
+	s, gf := database.GetGridFs("files")
+	defer s.Close()
+
+	// 被克隆爬虫的GridFS文件
+	f, err := gf.OpenId(spider.FileId)
+	if err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 新爬虫的GridFS文件
+	fNew, err := gf.Create(newSpider.Name + ".zip")
+	if err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 生成唯一ID
+	randomId := uuid.NewV4()
+	tmpPath := viper.GetString("other.tmppath")
+	if !utils.Exists(tmpPath) {
+		if err := os.MkdirAll(tmpPath, 0777); err != nil {
+			log.Errorf("mkdir other.tmppath error: %v", err.Error())
+			return err
+		}
+	}
+
+	// 创建临时文件
+	tmpFilePath := filepath.Join(tmpPath, randomId.String()+".zip")
+	tmpFile := utils.OpenFile(tmpFilePath)
+
+	// 拷贝到临时文件
+	if _, err := io.Copy(tmpFile, f); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 关闭临时文件
+	if err := tmpFile.Close(); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 读取内容
+	fContent, err := ioutil.ReadFile(tmpFilePath)
+	if err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 写入GridFS文件
+	if _, err := fNew.Write(fContent); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 关闭被克隆爬虫GridFS文件
+	if err = f.Close(); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 将新爬虫文件复制
+	newSpider.FileId = fNew.Id().(bson.ObjectId)
+
+	// 保存新爬虫
+	if err := newSpider.Add(); err != nil {
+		return err
+	}
+
+	// 关闭新爬虫GridFS文件
+	if err := fNew.Close(); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 删除临时文件
+	if err := os.RemoveAll(tmpFilePath); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	// 同步爬虫
+	PublishSpider(newSpider)
+
+	return nil
+}
+
 func CopySpider(spider model.Spider, newName string) error {
+	// 克隆GridFS文件
+	if err := cloneGridFsFile(spider, newName); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // 启动爬虫服务

backend/services/task.go

@@ -536,7 +536,7 @@ func SpiderFileCheck(t model.Task, spider model.Spider) error {
 	// 判断爬虫文件是否存在
 	gfFile := model.GetGridFs(spider.FileId)
 	if gfFile == nil {
-		t.Error = "找不到爬虫文件，请重新上传"
+		t.Error = "cannot find spider files, please re-upload"
 		t.Status = constants.StatusError
 		t.FinishTs = time.Now()                                 // 结束时间
 		t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长

frontend/src/components/Common/CrawlConfirmDialog.vue

@@ -78,8 +78,6 @@
             <span style="margin-left: 5px">跳转到任务详情页</span>
           </div>
         </el-form-item>
-        <el-form-item>
-        </el-form-item>
       </el-form>
       <template slot="footer">
         <el-button type="plain" size="small" @click="$emit('close')">{{$t('Cancel')}}</el-button>

frontend/src/components/Spider/CopySpiderDialog.vue

@@ -0,0 +1,85 @@
+<template>
+  <el-dialog
+    class="copy-spider-dialog"
+    ref="form"
+    :title="$t('Copy Spider')"
+    :visible="visible"
+    width="580px"
+    :before-close="onClose"
+  >
+    <el-form
+      label-width="160px"
+      :model="form"
+      ref="form"
+    >
+      <el-form-item
+        :label="$t('New Spider Name')"
+        required
+      >
+        <el-input v-model="form.name" :placeholder="$t('New Spider Name')"/>
+      </el-form-item>
+    </el-form>
+    <template slot="footer">
+      <el-button type="plain" size="small" @click="$emit('close')">{{$t('Cancel')}}</el-button>
+      <el-button
+        type="primary"
+        size="small"
+        :icon="isLoading ? 'el-icon-loading' : ''"
+        :disabled="isLoading"
+        @click="onConfirm"
+      >
+        {{$t('Confirm')}}
+      </el-button>
+    </template>
+  </el-dialog>
+</template>
+
+<script>
+export default {
+  name: 'CopySpiderDialog',
+  props: {
+    spiderId: {
+      type: String,
+      default: ''
+    },
+    visible: {
+      type: Boolean,
+      default: false
+    }
+  },
+  data () {
+    return {
+      form: {
+        name: ''
+      },
+      isLoading: false
+    }
+  },
+  methods: {
+    onClose () {
+      this.$emit('close')
+    },
+    onConfirm () {
+      this.$refs['form'].validate(async valid => {
+        if (!valid) return
+        try {
+          this.isLoading = true
+          const res = await this.$request.post(`/spiders/${this.spiderId}/copy`, this.form)
+          if (!res.data.error) {
+            this.$message.success('Copied successfully')
+          }
+          this.$emit('confirm')
+          this.$emit('close')
+          this.$st.sendEv('爬虫复制', '确认提交')
+        } finally {
+          this.isLoading = false
+        }
+      })
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

frontend/src/i18n/zh.js

@@ -220,6 +220,8 @@ export default {
   'Item Name': 'Item 名称',
   'Add Item': '添加 Item',
   'Add Variable': '添加变量',
+  'Copy Spider': '复制爬虫',
+  'New Spider Name': '新爬虫名称',
 
   // 爬虫列表
   'Name': '名称',
@@ -564,6 +566,7 @@ docker run -d --restart always --name crawlab_worker \\
   'Are you sure to delete selected items?': '您是否确认删除所选项？',
   'Are you sure to stop selected items?': '您是否确认停止所选项？',
   'Sent signals to cancel selected tasks': '已经向所选任务发送取消任务信号',
+  'Copied successfully': '已成功复制',
 
   // 其他
   'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'

frontend/src/views/spider/SpiderList.vue

@@ -290,6 +290,15 @@
     />
     <!--./crawl confirm dialog-->
 
+    <!--copy dialog-->
+    <copy-spider-dialog
+      :visible="copyDialogVisible"
+      :spider-id="activeSpiderId"
+      @close="copyDialogVisible = false"
+      @confirm="onCopyConfirm"
+    />
+    <!--./copy dialog-->
+
     <el-card style="border-radius: 0">
       <!--filter-->
       <div class="filter">
@@ -566,7 +575,7 @@
           >
           </el-table-column>
         </template>
-        <el-table-column :label="$t('Action')" align="left" fixed="right" min-width="170px">
+        <el-table-column :label="$t('Action')" align="left" fixed="right" min-width="220px">
           <template slot-scope="scope">
             <el-tooltip :content="$t('View')" placement="top">
               <el-button type="primary" icon="el-icon-search" size="mini"
@@ -576,6 +585,14 @@
               <el-button type="danger" icon="el-icon-delete" size="mini"
                          @click="onRemove(scope.row, $event)"></el-button>
             </el-tooltip>
+            <el-tooltip :content="$t('Copy')" placement="top">
+              <el-button
+                type="info"
+                icon="el-icon-copy-document"
+                size="mini"
+                @click="onCopy(scope.row, $event)"
+              />
+            </el-tooltip>
             <el-tooltip v-if="!isShowRun(scope.row)" :content="$t('No command line')" placement="top">
               <el-button disabled type="success" icon="fa fa-bug" size="mini"
                          @click="onCrawl(scope.row, $event)"></el-button>
@@ -619,10 +636,12 @@ import dayjs from 'dayjs'
 import CrawlConfirmDialog from '../../components/Common/CrawlConfirmDialog'
 import StatusTag from '../../components/Status/StatusTag'
 import StatusLegend from '../../components/Status/StatusLegend'
+import CopySpiderDialog from '../../components/Spider/CopySpiderDialog'
 
 export default {
   name: 'SpiderList',
   components: {
+    CopySpiderDialog,
     StatusLegend,
     CrawlConfirmDialog,
     StatusTag
@@ -784,7 +803,8 @@ export default {
       selectedSpiders: [],
       isStopLoading: false,
       isRemoveLoading: false,
-      isMultiple: false
+      isMultiple: false,
+      copyDialogVisible: false
     }
   },
   computed: {
@@ -966,6 +986,17 @@ export default {
         this.getList()
       }, 1000)
     },
+    onCopy (row, ev) {
+      ev.stopPropagation()
+      this.copyDialogVisible = true
+      this.activeSpiderId = row._id
+      this.$st.sendEv('爬虫列表', '点击复制')
+    },
+    onCopyConfirm () {
+      setTimeout(() => {
+        this.getList()
+      }, 1000)
+    },
     onView (row, ev) {
       ev.stopPropagation()
       this.$router.push('/spiders/' + row._id)