diff --git a/backend/model/spider.go b/backend/model/spider.go index 1fe0617f..a06e682b 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -4,6 +4,7 @@ import ( "crawlab/constants" "crawlab/database" "crawlab/entity" + "crawlab/utils" "errors" "github.com/apex/log" "github.com/globalsign/mgo" @@ -182,7 +183,7 @@ func GetSpider(id bson.ObjectId) (Spider, error) { } // 如果为可配置爬虫,获取爬虫配置 - if spider.Type == constants.Configurable { + if spider.Type == constants.Configurable && utils.Exists(filepath.Join(spider.Src, "Spiderfile")) { config, err := GetConfigSpiderData(spider) if err != nil { return spider, err @@ -229,10 +230,12 @@ func RemoveSpider(id bson.ObjectId) error { s, gf := database.GetGridFs("files") defer s.Close() - if err := gf.RemoveId(result.FileId); err != nil { - log.Error("remove file error, id:" + result.FileId.Hex()) - debug.PrintStack() - return err + if result.FileId.Hex() != constants.ObjectIdNull { + if err := gf.RemoveId(result.FileId); err != nil { + log.Error("remove file error, id:" + result.FileId.Hex()) + debug.PrintStack() + return err + } } return nil diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go index 68998737..982af28d 100644 --- a/backend/routes/config_spider.go +++ b/backend/routes/config_spider.go @@ -58,6 +58,23 @@ func PutConfigSpider(c *gin.Context) { } spider.Src = spiderDir + // 复制Spiderfile模版 + contentByte, err := ioutil.ReadFile("./template/Spiderfile") + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + f, err := os.Create(filepath.Join(spider.Src, "Spiderfile")) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + defer f.Close() + if _, err := f.Write(contentByte); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + // 添加爬虫到数据库 if err := spider.Add(); err != nil { HandleError(http.StatusInternalServerError, c, err)