crawlab/backend/mock/spider.go

package mock

import (
	"crawlab/model"
	"github.com/apex/log"
	"github.com/gin-gonic/gin"
	"github.com/globalsign/mgo/bson"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"time"
)

var SpiderList = []model.Spider{
	{
		Id:          bson.ObjectId("5d429e6c19f7abede924fee2"),
		Name:        "For test",
		DisplayName: "test",
		Type:        "test",
		Col:         "test",
		Site:        "www.baidu.com",
		Envs:        nil,
		Src:         "../app/spiders",
		Cmd:         "scrapy crawl test",
		LastRunTs:   time.Now(),
		CreateTs:    time.Now(),
		UpdateTs:    time.Now(),
	},
}

func GetSpiderList(c *gin.Context) {

	// mock get spider list from database
	results := SpiderList

	c.JSON(http.StatusOK, Response{
		Status:  "ok",
		Message: "success",
		Data:    results,
	})
}

func GetSpider(c *gin.Context) {
	id := c.Param("id")
	var result model.Spider

	if !bson.IsObjectIdHex(id) {
		HandleErrorF(http.StatusBadRequest, c, "invalid id")
	}

	for _, spider := range SpiderList {
		if spider.Id == bson.ObjectId(id) {
			result = spider
		}
	}

	c.JSON(http.StatusOK, Response{
		Status:  "ok",
		Message: "success",
		Data:    result,
	})
}

func PostSpider(c *gin.Context) {
	id := c.Param("id")

	if !bson.IsObjectIdHex(id) {
		HandleErrorF(http.StatusBadRequest, c, "invalid id")
	}

	var item model.Spider
	if err := c.ShouldBindJSON(&item); err != nil {
		HandleError(http.StatusBadRequest, c, err)
		return
	}

	log.Info("modify the item")

	c.JSON(http.StatusOK, Response{
		Status:  "ok",
		Message: "success",
	})
}
func GetSpiderDir(c *gin.Context) {
	// 爬虫ID
	id := c.Param("id")

	// 目录相对路径
	path := c.Query("path")
	var spi model.Spider

	// 获取爬虫
	for _, spider := range SpiderList {
		if spider.Id == bson.ObjectId(id) {
			spi = spider
		}
	}

	// 获取目录下文件列表
	f, err := ioutil.ReadDir(filepath.Join(spi.Src, path))
	if err != nil {
		HandleError(http.StatusInternalServerError, c, err)
		return
	}

	// 遍历文件列表
	var fileList []model.File
	for _, file := range f {
		fileList = append(fileList, model.File{
			Name:  file.Name(),
			IsDir: file.IsDir(),
			Size:  file.Size(),
			Path:  filepath.Join(path, file.Name()),
		})
	}

	// 返回结果
	c.JSON(http.StatusOK, Response{
		Status:  "ok",
		Message: "success",
		Data:    fileList,
	})
}

func GetSpiderTasks(c *gin.Context) {
	id := c.Param("id")

	var spider model.Spider
	for _, spi := range SpiderList {
		if spi.Id == bson.ObjectId(id) {
			spider = spi
		}
	}

	var tasks model.Task
	for _, task := range TaskList {
		if task.SpiderId == spider.Id {
			tasks = task
		}
	}

	c.JSON(http.StatusOK, Response{
		Status:  "ok",
		Message: "success",
		Data:    tasks,
	})
}

func DeleteSpider(c *gin.Context) {
	id := c.Param("id")

	if !bson.IsObjectIdHex(id) {
		HandleErrorF(http.StatusBadRequest, c, "invalid id")
		return
	}

	// 获取该爬虫,get this spider
	var spider model.Spider
	for _, spi := range SpiderList {
		if spi.Id == bson.ObjectId(id) {
			spider = spi
		}
	}

	// 删除爬虫文件目录,delete the spider dir
	if err := os.RemoveAll(spider.Src); err != nil {
		HandleError(http.StatusInternalServerError, c, err)
		return
	}

	// 从数据库中删除该爬虫,delete this spider from database

	c.JSON(http.StatusOK, Response{
		Status:  "ok",
		Message: "success",
	})
}