git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: enhance data model associations and remove unused components

Browse Source
This commit is contained in:
Marvin Zhang
2025-06-09 23:00:47 +08:00
parent c48d74bf0e
commit 1f3a9586b3
17 changed files with 334 additions and 505 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
core/controllers/schedule.go
View File

@@ -2,8 +2,6 @@ package controllers
import (
errors2 "errors"
mongo2 "github.com/crawlab-team/crawlab/core/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"github.com/crawlab-team/crawlab/core/interfaces"
@@ -45,26 +43,11 @@ func GetScheduleById(_ *gin.Context, params *GetByIdParams) (response *Response[
func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Schedule], err error) {
query := ConvertToBsonMFromListParams(params)
sort, err := GetSortOptionFromString(params.Sort)
if err != nil {
return GetErrorListResponse[models.Schedule](errors.BadRequestf("invalid request parameters: %v", err))
}
schedules, err := service.NewModelService[models.Schedule]().GetMany(query, &mongo2.FindOptions{
Sort: sort,
Skip: params.Size * (params.Page - 1),
Limit: params.Size,
})
if err != nil {
if !errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorListResponse[models.Schedule](err)
}
return GetListResponse[models.Schedule]([]models.Schedule{}, 0)
}
if len(schedules) == 0 {
return GetListResponse[models.Schedule]([]models.Schedule{}, 0)
}
skip, limit := GetSkipLimitFromListParams(params)
// total count
total, err := service.NewModelService[models.Schedule]().Count(query)
@@ -72,46 +55,23 @@ func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListRespo
return GetErrorListResponse[models.Schedule](err)
}
// ids
var ids []primitive.ObjectID
var spiderIds []primitive.ObjectID
for _, s := range schedules {
ids = append(ids, s.Id)
if !s.SpiderId.IsZero() {
spiderIds = append(spiderIds, s.SpiderId)
}
// check total
if total == 0 {
return GetEmptyListResponse[models.Schedule]()
}
// spider dict cache
var spiders []models.Spider
if len(spiderIds) > 0 {
spiders, err = service.NewModelService[models.Spider]().GetMany(bson.M{"_id": bson.M{"$in": spiderIds}}, nil)
if err != nil {
return GetErrorListResponse[models.Schedule](err)
}
}
dictSpider := map[primitive.ObjectID]models.Spider{}
for _, p := range spiders {
dictSpider[p.Id] = p
// aggregation pipelines
pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
// perform query
var schedules []models.Schedule
err = service.GetCollection[models.Schedule]().Aggregate(pipelines, nil).All(&schedules)
if err != nil {
return GetErrorListResponse[models.Schedule](err)
}
// iterate list again
var data []models.Schedule
for _, s := range schedules {
// spider
if !s.SpiderId.IsZero() {
p, ok := dictSpider[s.SpiderId]
if ok {
s.Spider = &p
}
}
// add to list
data = append(data, s)
}
// response
return GetListResponse(data, total)
return GetListResponse(schedules, total)
}
type PostScheduleParams struct {

194
core/controllers/spider.go
View File

@@ -1,7 +1,6 @@
package controllers
import (
"math"
"mime/multipart"
"os"
"path/filepath"
@@ -81,39 +80,14 @@ func GetSpiderById(_ *gin.Context, params *GetByIdParams) (response *Response[mo
}
// GetSpiderList handles getting a list of spiders with optional stats
func GetSpiderList(c *gin.Context, params *GetListParams) (response *ListResponse[models.Spider], err error) {
// get list
withStats := c.Query("stats")
if withStats == "" {
return getSpiderList(params)
}
// get list with stats
return getSpiderListWithStats(params)
}
func getSpiderList(params *GetListParams) (response *ListResponse[models.Spider], err error) {
func GetSpiderList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Spider], err error) {
// query parameters
query := ConvertToBsonMFromListParams(params)
sort, err := GetSortOptionFromString(params.Sort)
if err != nil {
return GetErrorListResponse[models.Spider](errors.BadRequestf("invalid request parameters: %v", err))
}
spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{
Sort: sort,
Skip: params.Size * (params.Page - 1),
Limit: params.Size,
})
if err != nil {
if !errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorListResponse[models.Spider](err)
}
return GetListResponse[models.Spider]([]models.Spider{}, 0)
}
if len(spiders) == 0 {
return GetListResponse[models.Spider]([]models.Spider{}, 0)
}
skip, limit := GetSkipLimitFromListParams(params)
// total count
total, err := service.NewModelService[models.Spider]().Count(query)
@@ -121,161 +95,29 @@ func getSpiderList(params *GetListParams) (response *ListResponse[models.Spider]
return GetErrorListResponse[models.Spider](err)
}
// ids
var ids []primitive.ObjectID
var gitIds []primitive.ObjectID
var projectIds []primitive.ObjectID
for _, s := range spiders {
ids = append(ids, s.Id)
if !s.GitId.IsZero() {
gitIds = append(gitIds, s.GitId)
}
if !s.ProjectId.IsZero() {
projectIds = append(projectIds, s.ProjectId)
}
// check total
if total == 0 {
return GetEmptyListResponse[models.Spider]()
}
// project dict cache
var projects []models.Project
if len(projectIds) > 0 {
projects, err = service.NewModelService[models.Project]().GetMany(bson.M{"_id": bson.M{"$in": projectIds}}, nil)
if err != nil {
return GetErrorListResponse[models.Spider](err)
}
}
dictProject := map[primitive.ObjectID]models.Project{}
for _, p := range projects {
dictProject[p.Id] = p
// aggregation pipelines
pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
pipelines = append(pipelines, service.GetJoinPipeline[models.SpiderStat]("_id", "_id", "_stat")...)
pipelines = append(pipelines, service.GetJoinPipeline[models.Task]("_stat.last_task_id", "_id", "_last_task")...)
pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_last_task._id", "_id", "_last_task._stat")...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Project]()...)
if utils.IsPro() {
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Git]()...)
}
// git dict cache
var gits []models.Git
if len(gitIds) > 0 && utils.IsPro() {
gits, err = service.NewModelService[models.Git]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil)
if err != nil {
return GetErrorListResponse[models.Spider](err)
}
}
dictGit := map[primitive.ObjectID]models.Git{}
for _, g := range gits {
dictGit[g.Id] = g
}
// iterate list again
var data []models.Spider
for _, s := range spiders {
// project
if !s.ProjectId.IsZero() {
p, ok := dictProject[s.ProjectId]
if ok {
s.Project = &p
}
}
// git
if !s.GitId.IsZero() && utils.IsPro() {
g, ok := dictGit[s.GitId]
if ok {
s.Git = &g
}
}
// add to list
data = append(data, s)
}
// response
return GetListResponse(data, total)
}
func getSpiderListWithStats(params *GetListParams) (response *ListResponse[models.Spider], err error) {
response, err = getSpiderList(params)
// perform query
var spiders []models.Spider
err = service.GetCollection[models.Spider]().Aggregate(pipelines, nil).All(&spiders)
if err != nil {
return GetErrorListResponse[models.Spider](err)
}
// spider ids
var ids []primitive.ObjectID
for _, s := range response.Data {
ids = append(ids, s.Id)
}
// spider stat dict
spiderStats, err := service.NewModelService[models.SpiderStat]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil)
if err != nil {
return GetErrorListResponse[models.Spider](err)
}
dictSpiderStat := map[primitive.ObjectID]models.SpiderStat{}
// task dict and task stat dict
var lastTasks []models.Task
var lastTaskIds []primitive.ObjectID
for _, st := range spiderStats {
if st.Tasks > 0 {
taskCount := int64(st.Tasks)
st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount)))
st.AverageRuntimeDuration = int64(math.Round(float64(st.RuntimeDuration) / float64(taskCount)))
st.AverageTotalDuration = int64(math.Round(float64(st.TotalDuration) / float64(taskCount)))
}
dictSpiderStat[st.Id] = st
if !st.LastTaskId.IsZero() {
lastTaskIds = append(lastTaskIds, st.LastTaskId)
}
}
dictLastTask := map[primitive.ObjectID]models.Task{}
dictLastTaskStat := map[primitive.ObjectID]models.TaskStat{}
if len(lastTaskIds) > 0 {
// task list
queryTask := bson.M{
"_id": bson.M{
"$in": lastTaskIds,
},
}
lastTasks, err = service.NewModelService[models.Task]().GetMany(queryTask, nil)
if err != nil {
return GetErrorListResponse[models.Spider](err)
}
// task stats list
taskStats, err := service.NewModelService[models.TaskStat]().GetMany(queryTask, nil)
if err != nil {
return GetErrorListResponse[models.Spider](err)
}
for _, st := range taskStats {
dictLastTaskStat[st.Id] = st
}
for _, t := range lastTasks {
st, ok := dictLastTaskStat[t.Id]
if ok {
t.Stat = &st
}
dictLastTask[t.SpiderId] = t
}
}
// iterate list again
for i, s := range response.Data {
// spider stat
st, ok := dictSpiderStat[s.Id]
if ok {
s.Stat = &st
}
// last task and stat
if !s.Stat.LastTaskId.IsZero() {
t, ok := dictLastTask[s.Stat.LastTaskId]
if ok {
s.Stat.LastTask = &t
}
}
response.Data[i] = s
}
return response, nil
return GetListResponse(spiders, total)
}
// PostSpider handles creating a new spider

154
core/controllers/task.go
View File

@@ -33,138 +33,62 @@ func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[
return GetErrorResponse[models.Task](err)
}
// task
t, err := service.NewModelService[models.Task]().GetById(id)
if errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorResponse[models.Task](err)
}
// aggregation pipelines
pipelines := service.GetByIdPipeline(id)
pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...)
// perform query
var tasks []models.Task
err = service.GetCollection[models.Task]().Aggregate(pipelines, nil).All(&tasks)
if err != nil {
return GetErrorResponse[models.Task](err)
}
// skip if task status is pending
if t.Status == constants.TaskStatusPending {
return GetDataResponse(*t)
}
// spider
if !t.SpiderId.IsZero() {
t.Spider, _ = service.NewModelService[models.Spider]().GetById(t.SpiderId)
}
// schedule
if !t.ScheduleId.IsZero() {
t.Schedule, _ = service.NewModelService[models.Schedule]().GetById(t.ScheduleId)
}
// node
if !t.NodeId.IsZero() {
t.Node, _ = service.NewModelService[models.Node]().GetById(t.NodeId)
}
// task stat
t.Stat, _ = service.NewModelService[models.TaskStat]().GetById(id)
return GetDataResponse(*t)
}
type GetTaskListParams struct {
*GetListParams
Stats bool `query:"stats"`
}
func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListResponse[models.Task], err error) {
if params.Stats {
return NewController[models.Task]().GetList(c, params.GetListParams)
}
// get query
query := ConvertToBsonMFromListParams(params.GetListParams)
sort, err := GetSortOptionFromString(params.GetListParams.Sort)
if err != nil {
return GetErrorListResponse[models.Task](err)
}
// get tasks
tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo2.FindOptions{
Sort: sort,
Skip: params.Size * (params.Page - 1),
Limit: params.Size,
})
if err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorListResponse[models.Task](err)
}
return GetErrorListResponse[models.Task](err)
}
// check empty list
// check results
if len(tasks) == 0 {
return GetListResponse[models.Task](nil, 0)
return nil, errors.NotFoundf("task %s not found", params.Id)
}
// ids
var taskIds []primitive.ObjectID
var spiderIds []primitive.ObjectID
for _, t := range tasks {
taskIds = append(taskIds, t.Id)
spiderIds = append(spiderIds, t.SpiderId)
}
return GetDataResponse(tasks[0])
}
// total count
func GetTaskList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Task], err error) {
// query parameters
query := ConvertToBsonMFromListParams(params)
sort, err := GetSortOptionFromString(params.Sort)
if err != nil {
return GetErrorListResponse[models.Task](err)
}
skip, limit := GetSkipLimitFromListParams(params)
// total
total, err := service.NewModelService[models.Task]().Count(query)
if err != nil {
return GetErrorListResponse[models.Task](err)
}
// stat list
stats, err := service.NewModelService[models.TaskStat]().GetMany(bson.M{
"_id": bson.M{
"$in": taskIds,
},
}, nil)
// check total
if total == 0 {
return GetEmptyListResponse[models.Task]()
}
// aggregation pipelines
pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...)
// perform query
var tasks []models.Task
err = service.GetCollection[models.Task]().Aggregate(pipelines, nil).All(&tasks)
if err != nil {
return GetErrorListResponse[models.Task](err)
}
// cache stat list to dict
statsDict := map[primitive.ObjectID]models.TaskStat{}
for _, s := range stats {
statsDict[s.Id] = s
}
// spider list
spiders, err := service.NewModelService[models.Spider]().GetMany(bson.M{
"_id": bson.M{
"$in": spiderIds,
},
}, nil)
if err != nil {
return GetErrorListResponse[models.Task](err)
}
// cache spider list to dict
spiderDict := map[primitive.ObjectID]models.Spider{}
for _, s := range spiders {
spiderDict[s.Id] = s
}
// iterate list again
for i, t := range tasks {
// task stat
ts, ok := statsDict[t.Id]
if ok {
tasks[i].Stat = &ts
}
// spider
s, ok := spiderDict[t.SpiderId]
if ok {
tasks[i].Spider = &s
}
}
return GetListResponse(tasks, total)
}

4
core/controllers/utils.go
View File

@@ -281,6 +281,10 @@ func SortsToOption(sorts []entity.Sort) (sort bson.D, err error) {
return sort, nil
}
func GetSkipLimitFromListParams(params *GetListParams) (skip int, limit int) {
return params.Size * (params.Page - 1), params.Size
}
type BaseResponse interface {
GetData() interface{}
GetDataString() string

11
core/models/models/spider.go
View File

@@ -21,18 +21,15 @@ type Spider struct {
Template string `json:"template,omitempty" bson:"template,omitempty" description:"Spider template"`
TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty" description:"Spider template params"`
// stats
Stat *SpiderStat `json:"stat,omitempty" bson:"-"`
// execution
Cmd string `json:"cmd" bson:"cmd" description:"Execute command"`
Param string `json:"param" bson:"param" description:"Default task param"`
Priority int `json:"priority" bson:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"`
// associated data
Project *Project `json:"project,omitempty" bson:"-"`
Git *Git `json:"git,omitempty" bson:"-"`
DataSource *Database `json:"data_source,omitempty" bson:"-"`
Stat *SpiderStat `json:"stat,omitempty" bson:"_stat,omitempty"`
LastTask *Task `json:"last_task,omitempty" bson:"_last_task,omitempty"`
Project *Project `json:"project,omitempty" bson:"_project,omitempty"`
Git *Git `json:"git,omitempty" bson:"_git,omitempty"`
}
type SpiderTemplateParams struct {

1
core/models/models/spider_stat.go
View File

@@ -8,7 +8,6 @@ type SpiderStat struct {
any `collection:"spider_stats"`
BaseModel `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty" description:"Last task ID"`
LastTask *Task `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks" description:"Task count"`
Results int `json:"results" bson:"results" description:"Result count"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty" description:"Wait duration (in second)"`

8
core/models/models/task.go
View File

@@ -20,8 +20,8 @@ type Task struct {
NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"`
// associated data
Stat *TaskStat `json:"stat,omitempty" bson:"-"`
Spider *Spider `json:"spider,omitempty" bson:"-"`
Schedule *Schedule `json:"schedule,omitempty" bson:"-"`
Node *Node `json:"node,omitempty" bson:"-"`
Stat *TaskStat `json:"stat,omitempty" bson:"_stat,omitempty"`
Node *Node `json:"node,omitempty" bson:"_node,omitempty"`
Spider *Spider `json:"spider,omitempty" bson:"_spider,omitempty"`
Schedule *Schedule `json:"schedule,omitempty" bson:"_schedule,omitempty"`
}

86
core/models/service/base_service.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/utils"
"reflect"
"sync"
@@ -303,13 +304,17 @@ func GetCollectionNameByInstance(v any) string {
return field.Tag.Get("collection")
}
func getCollectionName[T any]() string {
func GetCollectionName[T any]() string {
var instance T
t := reflect.TypeOf(instance)
field := t.Field(0)
return field.Tag.Get("collection")
}
func GetCollection[T any]() *mongo.Col {
return mongo.GetMongoCol(GetCollectionName[T]())
}
// NewModelService return singleton instance of ModelService
func NewModelService[T any]() *ModelService[T] {
typeName := fmt.Sprintf("%T", *new(T))
@@ -324,7 +329,7 @@ func NewModelService[T any]() *ModelService[T] {
var instance *ModelService[T]
onceMap[typeName].Do(func() {
collectionName := getCollectionName[T]()
collectionName := GetCollectionName[T]()
collection := mongo.GetMongoCol(collectionName)
instance = &ModelService[T]{col: collection}
instanceMap[typeName] = instance
@@ -351,3 +356,80 @@ func NewModelServiceWithColName[T any](colName string) *ModelService[T] {
return instanceMap[colName].(*ModelService[T])
}
func GetDefaultJoinPipeline[T any]() []bson.D {
return []bson.D{
GetDefaultLookupPipeline[T](),
GetDefaultUnwindPipeline[T](),
}
}
func GetJoinPipeline[T any](localField, foreignField, as string) []bson.D {
return []bson.D{
GetLookupPipeline[T](localField, foreignField, as),
GetUnwindPipeline(as),
}
}
func GetDefaultLookupPipeline[T any]() bson.D {
var model T
typ := reflect.TypeOf(model)
name := utils.ToSnakeCase(typ.Name())
return GetLookupByNamePipeline[T](name)
}
func GetLookupByNamePipeline[T any](name string) bson.D {
localField := fmt.Sprintf("%s_id", name)
foreignField := "_id"
as := fmt.Sprintf("_%s", name)
return GetLookupPipeline[T](localField, foreignField, as)
}
func GetLookupPipeline[T any](localField, foreignField, as string) bson.D {
return bson.D{{
Key: "$lookup",
Value: bson.M{
"from": GetCollectionName[T](),
"localField": localField,
"foreignField": foreignField,
"as": as,
}},
}
}
func GetDefaultUnwindPipeline[T any]() bson.D {
var model T
typ := reflect.TypeOf(model)
name := utils.ToSnakeCase(typ.Name())
as := fmt.Sprintf("_%s", name)
return GetUnwindPipeline(as)
}
func GetUnwindPipeline(as string) bson.D {
return bson.D{{
Key: "$unwind",
Value: bson.M{
"path": fmt.Sprintf("$%s", as),
"preserveNullAndEmptyArrays": true,
}},
}
}
func GetPaginationPipeline(query bson.M, sort bson.D, skip, limit int) []bson.D {
if query == nil {
query = bson.M{}
}
return []bson.D{
{{Key: "$match", Value: query}},
{{Key: "$sort", Value: sort}},
{{Key: "$skip", Value: skip}},
{{Key: "$limit", Value: limit}},
}
}
func GetByIdPipeline(id primitive.ObjectID) []bson.D {
return []bson.D{
{{Key: "$match", Value: bson.M{"_id": id}}},
{{Key: "$limit", Value: 1}},
}
}

38
frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue
View File

@@ -1,5 +1,5 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue';
import { computed, onBeforeMount, ref, watch } from 'vue';
import { useStore } from 'vuex';
import { useSpider, useProject, useNode } from '@/components';
import { TASK_MODE_RANDOM, TASK_MODE_SELECTED_NODES } from '@/constants/task';
@@ -9,6 +9,7 @@ import { useSpiderDetail } from '@/views';
import { getToRunNodes, priorityOptions, translate } from '@/utils';
import { getSpiderTemplateGroups, getSpiderTemplates } from '@/utils/spider';
import useRequest from '@/services/request';
import ClRemoteSelect from '@/components/ui/select/RemoteSelect.vue';
// i18n
const t = translate;
@@ -93,25 +94,6 @@ const activeTemplateOption = computed<SpiderTemplate | undefined>(() => {
return getSpiderTemplates().find(d => d.name === form.value.template);
});
const projectLoading = ref(false);
const projectList = ref<Project[]>([]);
const getProjects = async (query?: string) => {
try {
projectLoading.value = true;
const res = await get<Project[]>('/projects', {
conditions: {
name: query,
},
limit: 1000,
});
projectList.value = res.data || [];
} catch (e) {
console.error('Error setting project loading state:', e);
} finally {
projectLoading.value = false;
}
};
const getActiveNodes = async () => {
await store.dispatch('node/getActiveNodes');
};
@@ -194,21 +176,7 @@ defineOptions({ name: 'ClSpiderForm' });
:label="t('components.spider.form.project')"
prop="project_id"
>
<el-select
v-model="form.project_id"
:disabled="isFormItemDisabled('project_id')"
filterable
id="project"
class="project"
popper-class="spider-form-project"
>
<cl-option
v-for="op in allProjectSelectOptions"
:key="op.value"
:label="op.label"
:value="op.value"
/>
</el-select>
<cl-remote-select v-model="form.project_id" endpoint="/projects" />
</cl-form-item>
<!-- ./Row -->

53
frontend/crawlab-ui/src/components/core/task/TaskForm.vue
View File

@@ -1,5 +1,5 @@
<script setup lang="ts">
import { computed, watch } from 'vue';
import { computed, onBeforeMount, ref, watch } from 'vue';
import { useRouter } from 'vue-router';
import { useStore } from 'vuex';
import { ElMessage, ElMessageBox } from 'element-plus';
@@ -26,7 +26,7 @@ const t = translate;
const store = useStore();
// use node
const { activeNodesSorted: activeNodes, allDict: allNodeDict } = useNode(store);
const { activeNodesSorted: activeNodes } = useNode(store);
const toRunNodes = computed(() => {
const { mode, node_ids } = form.value;
@@ -38,18 +38,9 @@ const runNode = computed(() => {
return activeNodes.value.find(n => n._id === node_id);
});
// use spider
const { allListSelectOptions: allSpiderSelectOptions } = useSpider(store);
// use task
const {
form,
formRef,
allSpiderDict,
modeOptions,
modeOptionsDict,
isFormItemDisabled,
} = useTask(store);
const { form, formRef, modeOptions, modeOptionsDict, isFormItemDisabled } =
useTask(store);
// use task detail
const { activeId, getForm } = useTaskDetail();
@@ -72,11 +63,6 @@ watch(
}
);
const getSpiderName = (id: string) => {
const spider = allSpiderDict.value.get(id) as Spider;
return spider?.name;
};
const getModeName = (id: string) => {
const op = modeOptionsDict.value.get(id) as SelectOption;
return op?.label;
@@ -102,6 +88,29 @@ const noScheduleId = computed<boolean>(() =>
isZeroObjectId(form.value?.schedule_id)
);
const spiderLoading = ref(false);
const spiderList = ref<Spider[]>([]);
const getSpiders = async (query?: string) => {
try {
spiderLoading.value = true;
const res = await get<Spider[]>('/spiders', {
filter: JSON.stringify({ name: query }),
});
spiderList.value = res.data || [];
} catch (e) {
console.error('Error setting spider loading state:', e);
} finally {
spiderLoading.value = false;
}
};
const spiderSelectOptions = computed<SelectOption[]>(() =>
spiderList.value.map(spider => ({
label: spider.name,
value: spider._id,
}))
);
onBeforeMount(getSpiders);
const validate = async () => {
await formRef.value?.validate();
};
@@ -129,7 +138,7 @@ defineOptions({ name: 'ClTaskForm' });
filterable
>
<el-option
v-for="op in allSpiderSelectOptions"
v-for="op in spiderSelectOptions"
:key="op.value"
:label="op.label"
:value="op.value"
@@ -137,8 +146,8 @@ defineOptions({ name: 'ClTaskForm' });
</el-select>
<cl-nav-link
v-else
:label="form.spider?.name || getSpiderName(form.spider_id!)"
:path="`/spiders/${form.spider_id}`"
:label="form.spider?.name"
:path="`/spiders/${form.spider?._id}`"
/>
</cl-form-item>
<!-- ./Row -->
@@ -152,7 +161,7 @@ defineOptions({ name: 'ClTaskForm' });
prop="node_id"
>
<cl-node-tag
:node="allNodeDict.get(form.node_id!)"
:node="form.node"
size="large"
clickable
@click="router.push(`/nodes/${form.node_id}`)"

4
frontend/crawlab-ui/src/components/index.ts
View File

@@ -187,6 +187,7 @@ import NotificationSettingTriggerSelect from './core/notification/setting/Notifi
import Option from './ui/select/Option.vue';
import ProjectForm from './core/project/ProjectForm.vue';
import RangePicker from './ui/date/RangePicker.vue';
import RemoteSelect from './ui/select/RemoteSelect.vue';
import ResizeHandle from './ui/resize/ResizeHandle.vue';
import ResultCell from './core/result/ResultCell.vue';
import ResultCellDialog from './core/result/ResultCellDialog.vue';
@@ -196,7 +197,6 @@ import RunScheduleDialog from './core/schedule/RunScheduleDialog.vue';
import RunSpiderDialog from './core/spider/RunSpiderDialog.vue';
import ScheduleCron from './core/schedule/ScheduleCron.vue';
import ScheduleForm from './core/schedule/ScheduleForm.vue';
import Select from './ui/select/Select.vue';
import SpiderForm from './core/spider/SpiderForm.vue';
import SpiderResultDataWithDatabase from './core/spider/SpiderResultDataWithDatabase.vue';
import SpiderStat from './core/spider/SpiderStat.vue';
@@ -443,6 +443,7 @@ export {
Option as ClOption,
ProjectForm as ClProjectForm,
RangePicker as ClRangePicker,
RemoteSelect as ClRemoteSelect,
ResizeHandle as ClResizeHandle,
ResultCell as ClResultCell,
ResultCellDialog as ClResultCellDialog,
@@ -452,7 +453,6 @@ export {
RunSpiderDialog as ClRunSpiderDialog,
ScheduleCron as ClScheduleCron,
ScheduleForm as ClScheduleForm,
Select as ClSelect,
SpiderForm as ClSpiderForm,
SpiderResultDataWithDatabase as ClSpiderResultDataWithDatabase,
SpiderStat as ClSpiderStat,

98
frontend/crawlab-ui/src/components/ui/select/RemoteSelect.vue Normal file
View File

@@ -0,0 +1,98 @@
<script setup lang="ts">
import { computed, onBeforeMount, ref, watch } from 'vue';
import useRequest from '@/services/request';
const props = withDefaults(
defineProps<{
modelValue?: string;
placeholder?: string;
disabled?: boolean;
filterable?: boolean;
remoteShowSuffix?: boolean;
endpoint: string;
labelKey?: string;
valueKey?: string;
limit?: number;
}>(),
{
remoteShowSuffix: true,
labelKey: 'name',
valueKey: '_id',
limit: 1000,
}
);
const emit = defineEmits<{
(e: 'change', value: string): void;
(e: 'clear'): void;
(e: 'update:model-value', value: string): void;
}>();
const { get } = useRequest();
const internalValue = ref<string | undefined>(props.modelValue);
watch(
() => props.modelValue,
() => {
internalValue.value = props.modelValue;
}
);
watch(internalValue, () =>
emit('update:model-value', internalValue.value || '')
);
const loading = ref(false);
const list = ref([]);
const remoteMethod = async (query?: string) => {
const { endpoint, labelKey, limit } = props;
try {
loading.value = true;
let filter: string | undefined = undefined;
if (query) {
filter = JSON.stringify([
{ key: labelKey, op: 'contains', value: query } as FilterConditionData,
]);
}
const sort = labelKey;
const res = await get(endpoint, { filter, limit, sort });
list.value = res.data || [];
} catch (e) {
console.error(e);
} finally {
loading.value = false;
}
};
const selectOptions = computed<SelectOption[]>(() =>
list.value.map(row => ({
label: row[props.labelKey],
value: row[props.valueKey],
}))
);
onBeforeMount(remoteMethod);
defineOptions({ name: 'ClRemoteSelect' });
</script>
<template>
<el-select
v-model="internalValue"
:placeholder="placeholder"
:filterable="filterable"
:disabled="disabled"
remote
:remote-method="remoteMethod"
:remote-show-suffix="remoteShowSuffix"
@change="(value: any) => emit('change', value)"
@clear="() => emit('clear')"
>
<el-option
v-for="(op, index) in selectOptions"
:key="index"
:label="op.label"
:value="op.value"
/>
<template #label>
<slot name="label" />
</template>
</el-select>
</template>

30
frontend/crawlab-ui/src/components/ui/select/Select.vue
View File

@@ -1,30 +0,0 @@
<script setup lang="ts">
defineProps<{
label?: string;
placeholder?: string;
filterable?: boolean;
popperClass?: string;
}>();
const emit = defineEmits<{
(e: 'change', value: any): void;
(e: 'clear'): void;
}>();
defineOptions({ name: 'ClSelect' });
</script>
<template>
<el-select
v-model="internalModelValue"
:placeholder="placeholder"
:filterable="filterable"
:popper-class="popperClass"
@change="(value: any) => emit('change', value)"
@clear="() => emit('clear')"
>
<slot />
<template #label>
<slot name="label" />
</template>
</el-select>
</template>

10
frontend/crawlab-ui/src/interfaces/models/spider.d.ts vendored
View File

@@ -18,20 +18,22 @@ export declare global {
description?: string;
update_ts?: string;
create_ts?: string;
last_task?: Task;
stat?: SpiderStat;
incremental_sync?: boolean;
auto_install?: boolean;
git_id?: string;
git_root_path?: string;
git?: Git;
template?: SpiderTemplateName;
template_params?: SpiderTemplateParams;
// associated data
stat?: SpiderStat;
last_task?: Task;
project?: Project;
git?: Git;
}
interface SpiderStat {
_id: number;
last_task?: Task;
tasks: number;
results: number;
wait_duration: number;

20
frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx
View File

@@ -52,12 +52,6 @@ const useSpiderList = () => {
const { allListSelectOptions: allProjectListSelectOptions } =
useProject(store);
// const allProjectList = computed<Project[]>(() => store.state.project.allList);
// all project dict
const allProjectDict = computed<Map<string, Project>>(
() => store.getters['project/allDict']
);
// nav actions
const navActions = computed<ListActionGroup[]>(() => [
@@ -138,10 +132,12 @@ const useSpiderList = () => {
icon: ['fa', 'project-diagram'],
width: '120',
value: (row: Spider) => {
if (!row.project_id) return;
const p = allProjectDict.value.get(row.project_id);
if (!row.project?._id) return;
return (
<ClNavLink label={p?.name} path={`/projects/${row.project_id}`} />
<ClNavLink
label={row.project.name}
path={`/projects/${row.project._id}`}
/>
);
},
hasFilter: true,
@@ -176,7 +172,7 @@ const useSpiderList = () => {
icon: ['fa', 'heartbeat'],
width: '120',
value: (row: Spider) => {
const { _id, status, error } = row.stat?.last_task || {};
const { _id, status, error } = row.last_task || {};
if (!status) return;
return (
<ClTaskStatus
@@ -195,7 +191,7 @@ const useSpiderList = () => {
icon: ['fa', 'clock'],
width: '160',
value: (row: Spider) => {
const time = row.stat?.last_task?.stat?.started_at;
const time = row.last_task?.stat?.started_at;
if (!time) return;
return <ClTime time={time} />;
},
@@ -338,7 +334,7 @@ const useSpiderList = () => {
} as UseListOptions<Spider>;
// init
setupListComponent(ns, store, ['node', 'project']);
setupListComponent(ns, store);
return {
...useList<Spider>(ns, store, opts),

6
frontend/crawlab-ui/src/views/task/detail/actions/groups/TaskDetailActionGroupNav.vue
View File

@@ -1,7 +1,7 @@
<script setup lang="ts">
import { useStore } from 'vuex';
import { useRouter } from 'vue-router';
import { useNode, useTask } from '@/components';
import { useTask } from '@/components';
import { isZeroObjectId, translate } from '@/utils';
const t = translate;
@@ -12,8 +12,6 @@ const store = useStore();
const { form } = useTask(store);
const { allDict: allNodeDict } = useNode(store);
defineOptions({ name: 'ClTaskDetailActionGroupNav' });
</script>
@@ -22,7 +20,7 @@ defineOptions({ name: 'ClTaskDetailActionGroupNav' });
<cl-nav-action-fa-icon :icon="['fa', 'compass']" />
<cl-nav-action-item v-if="!isZeroObjectId(form?.node_id)">
<cl-node-tag
:node="allNodeDict.get(form.node_id!)"
:node="form.node"
size="large"
no-label
clickable

54
frontend/crawlab-ui/src/views/task/list/useTaskList.tsx
View File

@@ -73,21 +73,6 @@ const useTaskList = () => {
}
};
// all node dict
const allNodeDict = computed<Map<string, CNode>>(
() => store.getters['node/allDict']
);
// all spider dict
const allSpiderDict = computed<Map<string, Spider>>(
() => store.getters['spider/allDict']
);
// all schedule dict
const allScheduleDict = computed<Map<string, Schedule>>(
() => store.getters['schedule/allDict']
);
// nav actions
const navActions = computed<ListActionGroup[]>(() => [
{
@@ -183,15 +168,14 @@ const useTaskList = () => {
icon: ['fa', 'server'],
width: '160',
value: (row: Task) => {
if (!row.node_id) return;
const node = allNodeDict.value.get(row.node_id);
const { node } = row;
if (!node) return;
return (
<ClNodeTag
node={node}
clickable
onClick={async () => {
await router.push(`/nodes/${node?._id}`);
await router.push(`/nodes/${node._id}`);
}}
/>
);
@@ -203,13 +187,10 @@ const useTaskList = () => {
icon: ['fa', 'spider'],
width: '160',
value: (row: Task) => {
if (!row.spider_id) return;
const spider = row.spider || allSpiderDict.value.get(row.spider_id);
const { spider } = row;
if (!spider) return;
return (
<ClNavLink
label={spider?.name}
path={`/spiders/${spider?._id}`}
/>
<ClNavLink label={spider.name} path={`/spiders/${spider._id}`} />
);
},
},
@@ -219,12 +200,12 @@ const useTaskList = () => {
icon: ['fa', 'clock'],
width: '160',
value: (row: Task) => {
if (!row.schedule_id) return;
const schedule = allScheduleDict.value.get(row.schedule_id);
const { schedule } = row;
if (!schedule) return;
return (
<ClNavLink
label={schedule?.name}
path={`/schedules/${schedule?._id}`}
label={schedule.name}
path={`/schedules/${schedule._id}`}
/>
);
},
@@ -244,13 +225,9 @@ const useTaskList = () => {
icon: ['fa', 'terminal'],
width: '160',
value: (row: Task) => {
return (
<ClTaskCommand
task={row}
spider={allSpiderDict.value?.get(row.spider_id as string)}
size="small"
/>
);
const { spider } = row;
if (!spider) return;
return <ClTaskCommand task={row} spider={spider} size="small" />;
},
},
{
@@ -290,7 +267,10 @@ const useTaskList = () => {
icon: ['fa', 'clock'],
width: '120',
value: (row: Task) => {
if (!row.stat?.started_at || row.stat?.started_at.startsWith('000')) {
if (
!row.stat?.started_at ||
row.stat?.started_at.startsWith('000')
) {
return;
}
return <ClTime time={row.stat?.started_at} />;
@@ -477,7 +457,7 @@ const useTaskList = () => {
} as UseListOptions<Task>;
// init
setupListComponent(ns, store, ['node', 'project', 'spider', 'schedule']);
setupListComponent(ns, store);
return {
...useList<Task>(ns, store, opts),