crawlab/backend/services/config_spider.go

package services

import (
	"crawlab/constants"
	"crawlab/entity"
	"crawlab/model"
	"crawlab/model/config_spider"
	"errors"
	"fmt"
	"strings"
)

func GenerateConfigSpiderFiles(spider model.Spider, configData entity.ConfigSpiderData) error {
	// 校验Spiderfile正确性
	if err := ValidateSpiderfile(configData); err != nil {
		return err
	}

	// 构造代码生成器
	generator := config_spider.ScrapyGenerator{
		Spider:     spider,
		ConfigData: configData,
	}

	// 生成代码
	if err := generator.Generate(); err != nil {
		return err
	}

	return nil
}

// 验证Spiderfile
func ValidateSpiderfile(configData entity.ConfigSpiderData) error {
	// 获取所有字段
	fields := config_spider.GetAllFields(configData)

	// 校验是否存在 start_url
	if configData.StartUrl == "" {
		return errors.New("spiderfile invalid: start_url is empty")
	}

	// 校验是否存在 start_stage
	if configData.StartStage == "" {
		return errors.New("spiderfile invalid: start_stage is empty")
	}

	// 校验是否存在 stages
	if len(configData.Stages) == 0 {
		return errors.New("spiderfile invalid: stages is empty")
	}

	// 校验stages
	dict := map[string]int{}
	for stageName, stage := range configData.Stages {
		// stage 名称不能为空
		if stageName == "" {
			return errors.New("spiderfile invalid: stage name is empty")
		}

		// stage 名称不能为保留字符串
		// NOTE: 如果有其他Engine，可以扩展，默认为Scrapy
		if configData.Engine == "" || configData.Engine == constants.EngineScrapy {
			if strings.Contains(constants.ScrapyProtectedStageNames, stageName) {
				return errors.New(fmt.Sprintf("spiderfile invalid: stage name '%s' is protected", stageName))
			}
		} else {
			return errors.New(fmt.Sprintf("spiderfile invalid: engine '%s' is not implemented", configData.Engine))
		}

		// stage 名称不能重复
		if dict[stageName] == 1 {
			return errors.New(fmt.Sprintf("spiderfile invalid: stage name '%s' is duplicated", stageName))
		}
		dict[stageName] = 1

		// stage 字段不能为空
		if len(stage.Fields) == 0 {
			return errors.New(fmt.Sprintf("spiderfile invalid: stage '%s' has no fields", stageName))
		}

		// 是否包含 next_stage
		hasNextStage := false

		// 遍历字段列表
		for _, field := range stage.Fields {
			// stage 的 next stage 只能有一个
			if field.NextStage != "" {
				if hasNextStage {
					return errors.New(fmt.Sprintf("spiderfile invalid: stage '%s' has more than 1 next_stage", stageName))
				}
				hasNextStage = true
			}

			// 字段里 css 和 xpath 只能包含一个
			if field.Css != "" && field.Xpath != "" {
				return errors.New(fmt.Sprintf("spiderfile invalid: field '%s' in stage '%s' has both css and xpath set which is prohibited", field.Name, stageName))
			}
		}

		// stage 里 page_css 和 page_xpath 只能包含一个
		if stage.PageCss != "" && stage.PageXpath != "" {
			return errors.New(fmt.Sprintf("spiderfile invalid: stage '%s' has both page_css and page_xpath set which is prohibited", stageName))
		}

		// stage 里 list_css 和 list_xpath 只能包含一个
		if stage.ListCss != "" && stage.ListXpath != "" {
			return errors.New(fmt.Sprintf("spiderfile invalid: stage '%s' has both list_css and list_xpath set which is prohibited", stageName))
		}

		// 如果 stage 的 is_list 为 true 但 list_css 为空，报错
		if stage.IsList && stage.ListCss == "" {
			return errors.New("spiderfile invalid: stage with is_list = true should have list_css being set")
		}
	}

	// 校验字段唯一性
	if !IsUniqueConfigSpiderFields(fields) {
		return errors.New("spiderfile invalid: fields not unique")
	}

	// 字段名称不能为保留字符串
	for _, field := range fields {
		if strings.Contains(constants.ScrapyProtectedFieldNames, field.Name) {
			return errors.New(fmt.Sprintf("spiderfile invalid: field name '%s' is protected", field.Name))
		}
	}

	return nil
}

func IsUniqueConfigSpiderFields(fields []entity.Field) bool {
	dict := map[string]int{}
	for _, field := range fields {
		if dict[field.Name] == 1 {
			return false
		}
		dict[field.Name] = 1
	}
	return true
}