git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: update filter parameter in API requests and improve component structure

Browse Source
This commit is contained in:
Marvin Zhang
2025-06-12 00:17:40 +08:00
parent 3552b7805b
commit 79b7e074e1
34 changed files with 258 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
core/controllers/schedule.go
View File

@@ -2,8 +2,6 @@ package controllers
import (
errors2 "errors"
"go.mongodb.org/mongo-driver/mongo"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
@@ -11,6 +9,7 @@ import (
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/gin-gonic/gin"
"github.com/juju/errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
@@ -20,25 +19,24 @@ func GetScheduleById(_ *gin.Context, params *GetByIdParams) (response *Response[
if err != nil {
return GetErrorResponse[models.Schedule](errors.BadRequestf("invalid id format"))
}
s, err := service.NewModelService[models.Schedule]().GetById(id)
if errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorResponse[models.Schedule](errors.NotFoundf("spider not found"))
}
// aggregation pipelines
pipelines := service.GetByIdPipeline(id)
pipelines = addSchedulePipelines(pipelines)
// perform query
var schedules []models.Schedule
err = service.GetCollection[models.Schedule]().Aggregate(pipelines, nil).All(&schedules)
if err != nil {
return GetErrorResponse[models.Schedule](err)
}
// spider
if !s.SpiderId.IsZero() {
s.Spider, err = service.NewModelService[models.Spider]().GetById(s.SpiderId)
if err != nil {
if !errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorResponse[models.Schedule](err)
}
}
// check results
if len(schedules) == 0 {
return nil, errors.NotFoundf("schedule %s not found", params.Id)
}
return GetDataResponse(*s)
return GetDataResponse(schedules[0])
}
func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Schedule], err error) {
@@ -62,7 +60,7 @@ func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListRespo
// aggregation pipelines
pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
pipelines = addSchedulePipelines(pipelines)
// perform query
var schedules []models.Schedule
@@ -240,3 +238,8 @@ func postScheduleRunFunc(params *PostScheduleRunParams, userId primitive.ObjectI
return GetDataResponse(taskIds)
}
func addSchedulePipelines(pipelines []bson.D) []bson.D {
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
return pipelines
}

75
core/controllers/spider.go
View File

@@ -28,55 +28,24 @@ func GetSpiderById(_ *gin.Context, params *GetByIdParams) (response *Response[mo
if err != nil {
return GetErrorResponse[models.Spider](errors.BadRequestf("invalid id format"))
}
s, err := service.NewModelService[models.Spider]().GetById(id)
if errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorResponse[models.Spider](errors.NotFoundf("spider not found"))
}
// aggregation pipelines
pipelines := service.GetByIdPipeline(id)
pipelines = addSpiderPipelines(pipelines)
// perform query
var spiders []models.Spider
err = service.GetCollection[models.Spider]().Aggregate(pipelines, nil).All(&spiders)
if err != nil {
return GetErrorResponse[models.Spider](err)
}
// stat
s.Stat, err = service.NewModelService[models.SpiderStat]().GetById(s.Id)
if err != nil {
if !errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorResponse[models.Spider](err)
}
// check results
if len(spiders) == 0 {
return nil, errors.NotFoundf("spider %s not found", params.Id)
}
// project
if !s.ProjectId.IsZero() {
s.Project, err = service.NewModelService[models.Project]().GetById(s.ProjectId)
if err != nil {
if !errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorResponse[models.Spider](err)
}
}
}
// data collection (compatible to old version)
if s.ColName == "" && !s.ColId.IsZero() {
col, err := service.NewModelService[models.DataCollection]().GetById(s.ColId)
if err != nil {
if !errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorResponse[models.Spider](err)
}
} else {
s.ColName = col.Name
}
}
// git
if utils.IsPro() && !s.GitId.IsZero() {
s.Git, err = service.NewModelService[models.Git]().GetById(s.GitId)
if err != nil {
if !errors.Is(err, mongo.ErrNoDocuments) {
return GetErrorResponse[models.Spider](err)
}
}
}
return GetDataResponse(*s)
return GetDataResponse(spiders[0])
}
// GetSpiderList handles getting a list of spiders with optional stats
@@ -102,13 +71,7 @@ func GetSpiderList(_ *gin.Context, params *GetListParams) (response *ListRespons
// aggregation pipelines
pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
pipelines = append(pipelines, service.GetJoinPipeline[models.SpiderStat]("_id", "_id", "_stat")...)
pipelines = append(pipelines, service.GetJoinPipeline[models.Task]("_stat.last_task_id", "_id", "_last_task")...)
pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_last_task._id", "_id", "_last_task._stat")...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Project]()...)
if utils.IsPro() {
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Git]()...)
}
pipelines = addSpiderPipelines(pipelines)
// perform query
var spiders []models.Spider
@@ -676,3 +639,15 @@ func getSpiderRootPathByContext(c *gin.Context) (rootPath string, err error) {
}
return utils.GetSpiderRootPath(s)
}
func addSpiderPipelines(pipelines []bson.D) []bson.D {
pipelines = append(pipelines, service.GetJoinPipeline[models.SpiderStat]("_id", "_id", "_stat")...)
pipelines = append(pipelines, service.GetJoinPipeline[models.Task]("_stat.last_task_id", "_id", "_last_task")...)
pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_last_task._id", "_id", "_last_task._stat")...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Project]()...)
if utils.IsPro() {
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Git]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Database]()...)
}
return pipelines
}

32
core/controllers/task.go
View File

@@ -35,10 +35,7 @@ func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[
// aggregation pipelines
pipelines := service.GetByIdPipeline(id)
pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...)
pipelines = addTaskPipelines(pipelines)
// perform query
var tasks []models.Task
@@ -64,6 +61,20 @@ func GetTaskList(_ *gin.Context, params *GetListParams) (response *ListResponse[
}
skip, limit := GetSkipLimitFromListParams(params)
// get spider ids if query is not nil
if query != nil {
spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{Limit: 100})
if err != nil {
query = nil // reset query if error occurs
} else {
spiderIds := make([]primitive.ObjectID, 0, len(spiders))
for _, spider := range spiders {
spiderIds = append(spiderIds, spider.Id)
}
query = bson.M{"spider_id": bson.M{"$in": spiderIds}} // rewrite query to filter by spider ids
}
}
// total
total, err := service.NewModelService[models.Task]().Count(query)
if err != nil {
@@ -77,10 +88,7 @@ func GetTaskList(_ *gin.Context, params *GetListParams) (response *ListResponse[
// aggregation pipelines
pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...)
pipelines = addTaskPipelines(pipelines)
// perform query
var tasks []models.Task
@@ -425,3 +433,11 @@ func GetTaskResults(c *gin.Context, params *GetSpiderResultsParams) (response *L
return GetListResponse(results, total)
}
func addTaskPipelines(pipelines []bson.D) []bson.D {
pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...)
return pipelines
}

2
core/models/models/schedule.go
View File

@@ -21,5 +21,5 @@ type Schedule struct {
Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"`
// associated data
Spider *Spider `json:"spider" bson:"-" description:"Spider"`
Spider *Spider `json:"spider" bson:"_spider" description:"Spider"`
}

3
core/models/models/spider.go
View File

@@ -11,8 +11,8 @@ type Spider struct {
ColId primitive.ObjectID `json:"col_id" bson:"col_id" description:"Data collection id" deprecated:"true"`
ColName string `json:"col_name,omitempty" bson:"col_name" description:"Data collection name"`
DbName string `json:"db_name,omitempty" bson:"db_name" description:"Database name"`
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id" description:"Data source id"`
Description string `json:"description" bson:"description" description:"Description"`
DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id" description:"Database Id"`
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id" description:"Project ID"`
Mode string `json:"mode" bson:"mode" description:"Default task mode" enum:"random,all,selected-nodes"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids" description:"Default node ids, used in selected-nodes mode"`
@@ -30,6 +30,7 @@ type Spider struct {
LastTask *Task `json:"last_task,omitempty" bson:"_last_task,omitempty"`
Project *Project `json:"project,omitempty" bson:"_project,omitempty"`
Git *Git `json:"git,omitempty" bson:"_git,omitempty"`
Database *Database `json:"database,omitempty" bson:"_database,omitempty"`
}
type SpiderTemplateParams struct {

4
core/task/stats/service.go
View File

@@ -112,7 +112,7 @@ func (svc *Service) getDatabaseServiceItem(taskId primitive.ObjectID) (item *dat
var dbSvc interfaces2.DatabaseService
if utils.IsPro() {
if dbRegSvc := database.GetDatabaseRegistryService(); dbRegSvc != nil {
dbSvc, err = dbRegSvc.GetDatabaseService(s.DataSourceId)
dbSvc, err = dbRegSvc.GetDatabaseService(s.DatabaseId)
if err != nil {
return nil, err
}
@@ -123,7 +123,7 @@ func (svc *Service) getDatabaseServiceItem(taskId primitive.ObjectID) (item *dat
item = &databaseServiceItem{
taskId: taskId,
spiderId: s.Id,
dbId: s.DataSourceId,
dbId: s.DatabaseId,
dbSvc: dbSvc,
tableName: s.ColName,
time: time.Now(),

45
frontend/crawlab-ui/src/components/core/schedule/ScheduleForm.vue
View File

@@ -2,11 +2,9 @@
import { useStore } from 'vuex';
import { ElMessage } from 'element-plus';
import { TASK_MODE_SELECTED_NODES } from '@/constants/task';
import useSchedule from '@/components/core/schedule/useSchedule';
import useSpider from '@/components/core/spider/useSpider';
import useNode from '@/components/core/node/useNode';
import useTask from '@/components/core/task/useTask';
import { useSchedule, useNode, ClRemoteSelect } from '@/components';
import { priorityOptions, translate } from '@/utils';
import { ref } from 'vue';
const t = translate;
@@ -24,10 +22,7 @@ const {
} = useSchedule(store);
// use node
const { activeNodesSorted: activeNodes } = useNode(store);
// use spider
const { allListSelectOptions: allSpiderSelectOptions } = useSpider(store);
const { allNodesSorted: allNodes } = useNode(store);
// on enabled change
const onEnabledChange = async (value: boolean) => {
@@ -40,6 +35,22 @@ const onEnabledChange = async (value: boolean) => {
}
await store.dispatch(`${ns}/getList`);
};
const spiderRef = ref<typeof ClRemoteSelect>();
const onSpiderChange = (spiderId: string) => {
if (!spiderId) return;
const payload = { ...form.value } as Schedule;
if (!spiderRef.value) return;
const spider: Spider = spiderRef.value.getSelectedItem();
if (!spider) return;
if (spider.cmd) payload.cmd = spider.cmd;
if (spider.param) payload.param = spider.param;
if (spider.mode) payload.mode = spider.mode;
if (spider.node_ids?.length) payload.node_ids = spider.node_ids;
if (spider.node_tags?.length) payload.node_tags = spider.node_tags;
store.commit(`${ns}/setForm`, payload);
};
defineOptions({ name: 'ClScheduleForm' });
</script>
@@ -71,18 +82,12 @@ defineOptions({ name: 'ClScheduleForm' });
prop="spider_id"
required
>
<el-select
<cl-remote-select
ref="spiderRef"
v-model="form.spider_id"
:disabled="isFormItemDisabled('spider_id')"
filterable
>
<el-option
v-for="op in allSpiderSelectOptions"
:key="op.value"
:label="op.label"
:value="op.value"
/>
</el-select>
endpoint="/spiders"
@change="onSpiderChange"
/>
</cl-form-item>
<!-- ./Row -->
@@ -196,7 +201,7 @@ defineOptions({ name: 'ClScheduleForm' });
:placeholder="t('components.schedule.form.selectedNodes')"
>
<el-option
v-for="n in activeNodes"
v-for="n in allNodes"
:key="n.key"
:value="n._id"
:label="n.name"

7
frontend/crawlab-ui/src/components/core/schedule/useSchedule.ts
View File

@@ -5,7 +5,6 @@ import useScheduleService from '@/services/schedule/scheduleService';
import { getDefaultFormComponentData } from '@/utils/form';
import { parseExpression } from 'cron-parser';
import { getModeOptions } from '@/utils/task';
import useSpider from '@/components/core/spider/useSpider';
import { translate } from '@/utils/i18n';
import useScheduleDetail from '@/views/schedule/detail/useScheduleDetail';
@@ -20,8 +19,6 @@ const useSchedule = (store: Store<RootStoreState>) => {
const ns = 'schedule';
const state = store.state[ns];
const { allDict: allSpiderDict } = useSpider(store);
// form
const form = computed<Schedule>(() => state.form);
@@ -56,9 +53,9 @@ const useSchedule = (store: Store<RootStoreState>) => {
() => {
if (activeId.value) return;
if (!form.value?.spider_id) return;
const spider = allSpiderDict.value.get(form.value?.spider_id);
if (!spider) return;
if (!form.value?.spider) return;
const payload = { ...form.value } as Schedule;
const spider = form.value.spider;
if (spider.cmd) payload.cmd = spider.cmd;
if (spider.param) payload.param = spider.param;
if (spider.mode) payload.mode = spider.mode;

1
frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue
View File

@@ -180,7 +180,6 @@ defineOptions({ name: 'ClSpiderForm' });
<cl-remote-select
v-model="form.project_id"
endpoint="/projects"
filterable
:empty-option="{
label: t('common.status.unassigned'),
value: EMPTY_OBJECT_ID,

16
frontend/crawlab-ui/src/components/core/task/TaskResultDataWithDatabase.vue
View File

@@ -21,23 +21,19 @@ const dataRef = ref<typeof ClDatabaseTableDetailData | null>(null);
// store
const store = useStore();
const { task: state } = store.state as RootStoreState;
const { form } = useTask(store);
const { allDict: allSpiderDict } = useSpider(store);
const spider = computed<Spider | undefined>(
() =>
allSpiderDict.value.get(form.value?.spider_id || EMPTY_OBJECT_ID) as Spider
);
const spider = computed<Spider | undefined>(() => state.form.spider);
const activeTable = ref<DatabaseTable>();
const getActiveTable = debounce(async () => {
if (!spider.value) return;
const { data_source_id, db_name, col_name } = spider.value;
if (!data_source_id || !col_name) return;
const { database_id, db_name, col_name } = spider.value;
if (!database_id || !col_name) return;
const res = await post<any, Promise<ResponseWithData>>(
`/databases/${data_source_id}/tables/metadata/get`,
`/databases/${database_id}/tables/metadata/get`,
{
database: db_name,
table: col_name,
@@ -107,7 +103,7 @@ defineOptions({ name: 'ClTaskResultDataWithDatabase' });
v-if="activeTable"
ref="dataRef"
:active-table="activeTable"
:active-id="spider?.data_source_id || EMPTY_OBJECT_ID"
:active-id="spider?.database_id || EMPTY_OBJECT_ID"
:database-name="spider?.db_name"
:filter="dataFilter"
:display-all-fields="displayAllFields"

14
frontend/crawlab-ui/src/components/core/task/useTask.ts
View File

@@ -1,30 +1,23 @@
import { useRoute } from 'vue-router';
import { computed } from 'vue';
import { Store } from 'vuex';
import useForm from '@/components/ui/form/useForm';
import { useForm } from '@/components';
import useTaskService from '@/services/task/taskService';
import { getDefaultFormComponentData } from '@/utils/form';
import useSpider from '@/components/core/spider/useSpider';
import {
getDefaultFormComponentData,
getModeOptions,
getModeOptionsDict,
getPriorityLabel,
} from '@/utils/task';
import { formatTimeAgo } from '@/utils/time';
} from '@/utils';
// form component data
const formComponentData = getDefaultFormComponentData<Task>();
const useTask = (store: Store<RootStoreState>) => {
const ns = 'task' as ListStoreNamespace;
const { task: state } = store.state as RootStoreState;
// options for default mode
const modeOptions = getModeOptions();
const modeOptionsDict = computed(() => getModeOptionsDict());
const { allDict: allSpiderDict } = useSpider(store);
// route
const route = useRoute();
@@ -33,7 +26,6 @@ const useTask = (store: Store<RootStoreState>) => {
return {
...useForm<Task>('task', store, useTaskService(store), formComponentData),
allSpiderDict,
id,
modeOptions,
modeOptionsDict,

2
frontend/crawlab-ui/src/components/ui/filter/FilterInput.vue
View File

@@ -1,6 +1,6 @@
<script setup lang="ts">
import { onBeforeMount, ref } from 'vue';
import { debounce } from '@/utils';
import { debounce } from 'lodash';
const props = defineProps<{
id?: string;

21
frontend/crawlab-ui/src/components/ui/form/useForm.ts
View File

@@ -87,24 +87,6 @@ export const useForm = <T extends BaseModel>(
};
provide<(d: any) => boolean>('fn:isEmptyForm', isEmptyForm);
// all list select options
const allListSelectOptions = computed<SelectOption[]>(
() => store.getters[`${ns}/allListSelectOptions`]
);
// all list select options with empty
const allListSelectOptionsWithEmpty = computed<SelectOption[]>(() =>
allListSelectOptions.value.concat({
label: t('common.status.unassigned'),
value: EMPTY_OBJECT_ID,
})
);
// all dict
const allDict = computed<Map<string, T>>(
() => store.getters[`${ns}/allDict`]
);
// services
const { getList, create, updateById } = services;
@@ -228,9 +210,6 @@ export const useForm = <T extends BaseModel>(
isFormItemDisabled,
activeDialogKey,
createEditDialogVisible,
allListSelectOptions,
allListSelectOptionsWithEmpty,
allDict,
confirmDisabled,
confirmLoading,
setConfirmLoading,

27
frontend/crawlab-ui/src/components/ui/select/RemoteSelect.vue
View File

@@ -1,12 +1,16 @@
<script setup lang="ts">
import { computed, onBeforeMount, ref, watch } from 'vue';
import useRequest from '@/services/request';
import { Placement } from '@popperjs/core';
import { FILTER_OP_CONTAINS } from '@/constants';
const props = withDefaults(
defineProps<{
modelValue?: string;
placeholder?: string;
disabled?: boolean;
size?: BasicSize;
placement?: Placement;
filterable?: boolean;
clearable?: boolean;
remoteShowSuffix?: boolean;
@@ -17,15 +21,17 @@ const props = withDefaults(
emptyOption?: SelectOption;
}>(),
{
filterable: true,
remoteShowSuffix: true,
labelKey: 'name',
valueKey: '_id',
limit: 1000,
limit: 100,
}
);
const emit = defineEmits<{
(e: 'change', value: string): void;
(e: 'select', value: string): void;
(e: 'clear'): void;
(e: 'update:model-value', value: string): void;
}>();
@@ -44,7 +50,7 @@ watch(internalValue, () =>
);
const loading = ref(false);
const list = ref([]);
const list = ref<any[]>([]);
const remoteMethod = async (query?: string) => {
const { endpoint, labelKey, limit } = props;
try {
@@ -52,7 +58,11 @@ const remoteMethod = async (query?: string) => {
let filter: string | undefined = undefined;
if (query) {
filter = JSON.stringify([
{ key: labelKey, op: 'contains', value: query } as FilterConditionData,
{
key: labelKey,
op: FILTER_OP_CONTAINS,
value: query,
} as FilterConditionData,
]);
}
const sort = labelKey;
@@ -81,16 +91,27 @@ const selectOptions = computed<SelectOption[]>(() => {
});
onBeforeMount(remoteMethod);
const getSelectedItem = () => {
const { valueKey } = props;
return list.value.find(item => item[valueKey] === internalValue.value);
};
defineExpose({
getSelectedItem,
});
defineOptions({ name: 'ClRemoteSelect' });
</script>
<template>
<el-select
v-model="internalValue"
:size="size"
:placeholder="placeholder"
:filterable="filterable"
:disabled="disabled"
:clearable="clearable"
:placement="placement"
remote
:remote-method="remoteMethod"
:remote-show-suffix="remoteShowSuffix"

5
frontend/crawlab-ui/src/components/ui/table/TableHeaderDialogFilter.vue
View File

@@ -1,9 +1,8 @@
<script setup lang="ts">
import { getDefaultFilterCondition } from '@/components/ui/filter/filter';
import { computed, ref, watch } from 'vue';
import { debounce } from '@/utils/debounce';
import { debounce } from 'lodash';
import { Search } from '@element-plus/icons-vue';
import { getDefaultFilterCondition } from '@/components/ui/filter/filter';
import { emptyArrayFunc, translate } from '@/utils';
const props = withDefaults(

3
frontend/crawlab-ui/src/interfaces/models/spider.d.ts vendored
View File

@@ -9,7 +9,7 @@ export declare global {
col_id?: string;
col_name?: string;
db_name?: string;
data_source_id?: string;
database_id?: string;
mode?: TaskMode;
node_ids?: string[];
node_tags?: string[];
@@ -30,6 +30,7 @@ export declare global {
last_task?: Task;
project?: Project;
git?: Git;
database?: Database;
}
interface SpiderStat {

4
frontend/crawlab-ui/src/interfaces/services/request.d.ts vendored
View File

@@ -2,8 +2,8 @@ export declare global {
interface ListRequestParams {
page?: number;
size?: number;
conditions?: FilterConditionData[] | string;
all?: boolean | string | number;
filter?: string;
sort?: string;
[key: string]: any;
}

3
frontend/crawlab-ui/src/interfaces/views/schedule.d.ts vendored
View File

@@ -12,4 +12,7 @@ interface Schedule {
node_ids?: string[];
node_tags?: string[];
enabled?: boolean;
// associated data
spider?: Spider;
}

27
frontend/crawlab-ui/src/layouts/content/detail/DetailLayout.vue
View File

@@ -1,5 +1,11 @@
<script setup lang="ts">
import { computed, onBeforeMount, onBeforeUnmount, onMounted } from 'vue';
import {
computed,
onBeforeMount,
onBeforeUnmount,
onMounted,
watch,
} from 'vue';
import { useStore } from 'vuex';
import { useDetail } from '@/layouts';
@@ -33,7 +39,7 @@ const {
onBack,
onSave,
tabs,
} = useDetail(ns.value);
} = useDetail(props.storeNamespace);
const computedTabs = computed<NavItem[]>(() =>
tabs.value.map((tab: NavItem) => ({ ...tab }))
@@ -42,8 +48,20 @@ const computedTabs = computed<NavItem[]>(() =>
// Fetch the form data when the component is mounted
onBeforeMount(getForm);
// Watch for changes in the activeId and fetch the form data accordingly
watch(
() => activeId.value,
async () => {
if (!activeId.value) return;
await getForm();
}
);
// Fetch navigation list before mounting the component
onBeforeMount(() => store.dispatch(`${ns.value}/getNavList`));
const getNavList = async (query?: string) => {
await store.dispatch(`${ns.value}/getNavList`, query);
};
onBeforeMount(getNavList);
// reset form before unmount
onBeforeUnmount(() => {
@@ -73,6 +91,9 @@ defineOptions({ name: 'ClDetailLayout' });
size="small"
placement="bottom-end"
filterable
remote
remote-show-suffix
:remote-method="getNavList"
@change="onNavSelect"
>
<el-option

17
frontend/crawlab-ui/src/layouts/content/detail/useDetail.ts
View File

@@ -1,10 +1,8 @@
import { useRoute, useRouter } from 'vue-router';
import { useStore } from 'vuex';
import { computed, watch, provide, ref } from 'vue';
import { getRoutePath, getTabName } from '@/utils/route';
import { ElMessage } from 'element-plus';
import { translate } from '@/utils/i18n';
import { debounce, isPro } from '@/utils';
import { getRoutePath, getTabName, isPro, translate } from '@/utils';
// i18n
const t = translate;
@@ -19,13 +17,11 @@ const useDetail = <T extends BaseModel>(ns: ListStoreNamespace) => {
const state = rootState[ns] as BaseStoreState;
const commonState = rootState.common;
const { form } = state;
const showActionsToggleTooltip = ref<boolean>(false);
const activeId = computed<string>(() => {
const { id } = route.params;
return (id as string) || form._id || '';
return (id as string) || state.form._id || '';
});
const navItems = computed<NavItem<T>[]>(() => {
@@ -38,6 +34,7 @@ const useDetail = <T extends BaseModel>(ns: ListStoreNamespace) => {
}) as NavItem<T>[];
if (!items.some(item => item.id === activeId.value)) {
// if activeId is not in navList, add it
const { form } = state;
items.unshift({
id: activeId.value,
label: form.name || activeId.value,
@@ -87,10 +84,9 @@ const useDetail = <T extends BaseModel>(ns: ListStoreNamespace) => {
const afterSave = computed<Function[]>(() => state.afterSave);
const getForm = debounce(async () => {
if (!activeId.value) return;
const getForm = async () => {
return await store.dispatch(`${ns}/getById`, activeId.value);
});
};
const onNavTabsSelect = async (tabName: string) => {
await router.push(`${primaryRoutePath.value}/${activeId.value}/${tabName}`);
@@ -115,9 +111,6 @@ const useDetail = <T extends BaseModel>(ns: ListStoreNamespace) => {
afterSave.value.map(fn => fn());
};
// get form when active id changes
watch(() => activeId.value, getForm);
// store context
provide<DetailStoreContext<T>>('store-context', {
namespace: ns,

5
frontend/crawlab-ui/src/services/index.ts
View File

@@ -1,5 +1,4 @@
import useRequest from '@/services/request';
import { debounce } from '@/utils';
import * as llmService from './llm';
// Export the LLM service
@@ -10,9 +9,9 @@ const { get, put, post, del, getList, putList, postList, delList } =
export const useService = <T = any>(endpoint: string): Services<T> => {
return {
getById: debounce(async (id: string) => {
getById: async (id: string) => {
return await get<T>(`${endpoint}/${id}`);
}) as any,
},
create: async (form: T) => {
return await post<{ data: T }, ResponseWithData<T>>(`${endpoint}`, {
data: form,

5
frontend/crawlab-ui/src/services/request.ts
View File

@@ -150,11 +150,6 @@ const useRequest = () => {
params?: ListRequestParams,
opts?: AxiosRequestConfig
) => {
// normalize conditions
if (params && Array.isArray(params.conditions)) {
params.conditions = JSON.stringify(params.conditions);
}
// get request
const res = await get<T, ResponseWithListData<T>, ListRequestParams>(
url,

2
frontend/crawlab-ui/src/store/modules/dependency.ts
View File

@@ -285,7 +285,7 @@ const actions = {
const res = await getList(`${endpoint}/repos`, {
page,
size,
conditions: JSON.stringify(tableListFilter),
filter: JSON.stringify(tableListFilter),
sort: JSON.stringify(tableListSort),
lang,
});

3
frontend/crawlab-ui/src/store/modules/node.ts
View File

@@ -45,8 +45,7 @@ const actions = {
const res = await get<Record<string, Metric>>('/nodes/metrics', {
page,
size,
conditions: JSON.stringify(state.tableListFilter),
// sort: JSON.stringify(state.tableListSort),
filter: JSON.stringify(state.tableListFilter),
} as ListRequestParams);
commit('setNodeMetricsMap', res.data);
return res;

4
frontend/crawlab-ui/src/utils/store.ts
View File

@@ -261,7 +261,7 @@ export const getDefaultStoreActions = <T = any>(
const res = await getList({
page,
size,
conditions: JSON.stringify(state.tableListFilter),
filter: JSON.stringify(state.tableListFilter),
sort: JSON.stringify(state.tableListSort),
} as ListRequestParams);
@@ -306,7 +306,7 @@ export const getDefaultStoreActions = <T = any>(
) => {
const res = await getList({
size: 100,
conditions: query
filter: query
? JSON.stringify([
{ key: 'name', op: FILTER_OP_CONTAINS, value: query },
] as FilterConditionData[])

3
frontend/crawlab-ui/src/views/git/detail/GitDetail.vue
View File

@@ -9,13 +9,14 @@ import {
} from 'vue';
import { useStore } from 'vuex';
import { useRouter } from 'vue-router';
import { debounce } from 'lodash';
import {
FILE_ROOT,
GIT_STATUS_READY,
TAB_NAME_CHANGES,
TAB_NAME_FILES,
} from '@/constants';
import { debounce, translate } from '@/utils';
import { translate } from '@/utils';
import useGitDetail from '@/views/git/detail/useGitDetail';
import useGit from '@/components/core/git/useGit';
import type { TagProps } from '@/components/ui/tag/types';

4
frontend/crawlab-ui/src/views/git/detail/tabs/GitDetailTabChanges.vue
View File

@@ -2,12 +2,14 @@
import { computed, h, ref, watch, onBeforeMount } from 'vue';
import { useStore } from 'vuex';
import { ElMessage } from 'element-plus';
import { debounce } from 'lodash';
import GitFileStatus from '@/components/core/git/GitFileStatus.vue';
import Tag from '@/components/ui/tag/Tag.vue';
import Table from '@/components/ui/table/Table.vue';
import useGitDetail from '@/views/git/detail/useGitDetail';
import { debounce, translate } from '@/utils';
import { translate } from '@/utils';
import { TABLE_COLUMN_NAME_ACTIONS } from '@/constants';
import { TagProps } from '@/components/ui/tag/types';
// i18n
const t = translate;

3
frontend/crawlab-ui/src/views/git/detail/tabs/GitDetailTabCommits.vue
View File

@@ -2,7 +2,8 @@
import { computed, h, onBeforeMount, ref, watch } from 'vue';
import { useStore } from 'vuex';
import { Column } from 'element-plus';
import { debounce, translate } from '@/utils';
import { debounce } from 'lodash';
import { translate } from '@/utils';
import { GIT_REF_TYPE_BRANCH } from '@/constants/git';
import Time from '@/components/ui/time/Time.vue';
import Tag from '@/components/ui/tag/Tag.vue';

3
frontend/crawlab-ui/src/views/git/detail/tabs/GitDetailTabFiles.vue
View File

@@ -1,7 +1,8 @@
<script setup lang="ts">
import { inject, ref, watch } from 'vue';
import { useStore } from 'vuex';
import { debounce, translate } from '@/utils';
import { debounce } from 'lodash';
import { translate } from '@/utils';
import useGitService from '@/services/git/gitService';
import { useGitDetail } from '@/views';
import { ElMessage, ElMessageBox } from 'element-plus';

2
frontend/crawlab-ui/src/views/notification/setting/detail/tabs/NotificationSettingDetailTabChannels.vue
View File

@@ -20,8 +20,6 @@ const { form } = useNotificationSetting(store);
const { activeId } = useNotificationSettingDetail();
const { allListSelectOptions, allDict } = useNotificationChannel(store);
const selectAll = ref(false);
const selectIntermediate = ref(false);
const updateSelectAll = () => {

108
frontend/crawlab-ui/src/views/schedule/list/useScheduleList.tsx
View File

@@ -1,17 +1,23 @@
import { computed, h } from 'vue';
import { TABLE_COLUMN_NAME_ACTIONS } from '@/constants/table';
import { useStore } from 'vuex';
import { ElMessage, ElMessageBox } from 'element-plus';
import useList from '@/layouts/content/list/useList';
import NavLink from '@/components/ui/nav/NavLink.vue';
import { ElMessage } from 'element-plus';
import { useRouter } from 'vue-router';
import { onListFilterChangeByKey, setupListComponent } from '@/utils/list';
import TaskMode from '@/components/core/task/TaskMode.vue';
import ScheduleCron from '@/components/core/schedule/ScheduleCron.vue';
import Switch from '@/components/ui/switch/Switch.vue';
import useSpider from '@/components/core/spider/useSpider';
import useTask from '@/components/core/task/useTask';
import { translate } from '@/utils/i18n';
import {
ClNavLink,
ClTaskMode,
ClScheduleCron,
ClSwitch,
useTask,
} from '@/components';
import { useList } from '@/layouts';
import {
translate,
onListFilterChangeByKey,
setupListComponent,
getIconByAction,
isAllowedAction,
} from '@/utils';
import {
ACTION_ADD,
ACTION_DELETE,
@@ -28,7 +34,6 @@ import {
TASK_MODE_RANDOM,
TASK_MODE_SELECTED_NODES,
} from '@/constants';
import { getIconByAction, isAllowedAction } from '@/utils';
// i18n
const t = translate;
@@ -41,7 +46,6 @@ const useScheduleList = () => {
const ns = 'schedule';
const store = useStore<RootStoreState>();
const { commit } = store;
const { schedule: state } = store.state;
// use list
const { actionFunctions } = useList<Schedule>(ns, store);
@@ -49,13 +53,6 @@ const useScheduleList = () => {
// action functions
const { deleteByIdConfirm } = actionFunctions;
// all spider dict
const allSpiderDict = computed<Map<string, Spider>>(
() => store.getters['spider/allDict']
);
const { allListSelectOptions: allSpiderListSelectOptions } = useSpider(store);
const { modeOptions } = useTask(store);
// nav actions
@@ -178,11 +175,9 @@ const useScheduleList = () => {
label: t('views.schedules.table.columns.name'),
icon: ['fa', 'font'],
width: '150',
value: (row: Schedule) =>
h(NavLink, {
path: `/schedules/${row._id}`,
label: row.name,
}),
value: (row: Schedule) => (
<ClNavLink path={`/schedules/${row._id}`} label={row.name} />
),
hasSort: true,
hasFilter: true,
allowFilterSearch: true,
@@ -193,17 +188,12 @@ const useScheduleList = () => {
icon: ['fa', 'spider'],
width: '160',
value: (row: Schedule) => {
if (!row.spider_id) return;
const spider = allSpiderDict.value.get(row.spider_id);
return h(NavLink, {
label: spider?.name,
path: `/spiders/${spider?._id}`,
});
const { spider } = row;
if (!spider) return;
return (
<ClNavLink path={`/spiders/${spider._id}`} label={spider.name} />
);
},
hasFilter: true,
allowFilterSearch: true,
allowFilterItems: true,
filterItems: allSpiderListSelectOptions.value,
},
{
key: 'mode',
@@ -211,7 +201,7 @@ const useScheduleList = () => {
icon: ['fa', 'cog'],
width: '160',
value: (row: Schedule) => {
return h(TaskMode, { mode: row.mode });
return <ClTaskMode mode={row.mode} />;
},
hasFilter: true,
allowFilterItems: true,
@@ -223,7 +213,7 @@ const useScheduleList = () => {
icon: ['fa', 'clock'],
width: '160',
value: (row: Schedule) => {
return h(ScheduleCron, { cron: row.cron } as ScheduleCronProps);
return <ClScheduleCron cron={row.cron} />;
},
hasFilter: true,
allowFilterSearch: true,
@@ -234,27 +224,31 @@ const useScheduleList = () => {
icon: ['fa', 'toggle-on'],
width: '120',
value: (row: Schedule) => {
return h(Switch, {
modelValue: row.enabled,
disabled: !isAllowedAction(
router.currentRoute.value.path,
ACTION_ENABLE
),
'onUpdate:modelValue': async (value: boolean) => {
if (value) {
await store.dispatch(`${ns}/enable`, row._id);
ElMessage.success(
t('components.schedule.message.success.enable')
);
} else {
await store.dispatch(`${ns}/disable`, row._id);
ElMessage.success(
t('components.schedule.message.success.disable')
);
return (
<ClSwitch
modelValue={row.enabled}
disabled={
!isAllowedAction(
router.currentRoute.value.path,
ACTION_ENABLE
)
}
await store.dispatch(`${ns}/getList`);
},
} as SwitchProps);
onUpdate:modelValue={async (value: boolean) => {
if (value) {
await store.dispatch(`${ns}/enable`, row._id);
ElMessage.success(
t('components.schedule.message.success.enable')
);
} else {
await store.dispatch(`${ns}/disable`, row._id);
ElMessage.success(
t('components.schedule.message.success.disable')
);
}
await store.dispatch(`${ns}/getList`);
}}
/>
);
},
hasFilter: true,
allowFilterItems: true,
@@ -326,7 +320,7 @@ const useScheduleList = () => {
} as UseListOptions<Schedule>;
// init
setupListComponent(ns, store, ['node', 'spider']);
setupListComponent(ns, store);
return {
...useList<Schedule>(ns, store, opts),

32
frontend/crawlab-ui/src/views/spider/detail/actions/SpiderDetailActionsData.vue
View File

@@ -27,8 +27,6 @@ const form = computed(() => state.form);
const { activeId } = useSpiderDetail();
const { allDict: allDatabaseDict } = useDatabase(store);
const allDatabaseSelectOptions = computed<SelectOption[]>(() => {
// TODO: implement
return [];
@@ -49,8 +47,10 @@ const allDatabaseSelectOptions = computed<SelectOption[]>(() => {
// });
});
const currentDatabase = computed(() => {
return allDatabaseDict.value.get(dataSourceId.value) as Database | undefined;
const currentDatabase = computed<Database | undefined>(() => {
// TODO: implement
return undefined;
// return allDatabaseDict.value.get(databaseId.value) as Database | undefined;
});
const isDatabaseOffline = computed(() => {
@@ -84,9 +84,9 @@ watch(isMultiDatabases, () => {
}
});
const dataSourceId = ref<string>(form.value?.data_source_id || EMPTY_OBJECT_ID);
const databaseId = ref<string>(form.value?.database_id || EMPTY_OBJECT_ID);
const onDatabaseChange = async (value: string) => {
dataSourceId.value = form.value?.data_source_id || EMPTY_OBJECT_ID;
databaseId.value = form.value?.database_id || EMPTY_OBJECT_ID;
await ElMessageBox.confirm(
t('components.spider.messageBox.confirm.changeDatabase.message'),
t('components.spider.messageBox.confirm.changeDatabase.title'),
@@ -96,7 +96,7 @@ const onDatabaseChange = async (value: string) => {
);
store.commit(`${ns}/setForm`, {
...form.value,
data_source_id: value,
database_id: value,
});
try {
await store.dispatch(`${ns}/updateById`, {
@@ -109,15 +109,17 @@ const onDatabaseChange = async (value: string) => {
}
};
watch(
() => form.value?.data_source_id,
() => form.value?.database_id,
value => {
dataSourceId.value = value || EMPTY_OBJECT_ID;
databaseId.value = value || EMPTY_OBJECT_ID;
}
);
const getDataSourceByDatabaseId = (id: string): DatabaseDataSource => {
const db = allDatabaseDict.value.get(id) as Database | undefined;
if (!db?.data_source) return 'mongo';
return db.data_source;
// TODO: implement
return 'mongo';
// const db = allDatabaseDict.value.get(id) as Database | undefined;
// if (!db?.data_source) return 'mongo';
// return db.data_source;
};
// database table options
@@ -190,20 +192,20 @@ defineOptions({ name: 'ClSpiderDetailActionsData' });
<el-select
class="database"
:class="isDatabaseOffline ? 'offline' : ''"
v-model="dataSourceId"
v-model="databaseId"
@change="onDatabaseChange"
>
<template #label="{ label }">
<div>
<cl-database-data-source
:data-source="
getDataSourceByDatabaseId(form.data_source_id as string)
getDataSourceByDatabaseId(form.database_id as string)
"
icon-only
/>
<span style="margin: 5px">{{ label }}</span>
<cl-icon
v-if="form.data_source_id === EMPTY_OBJECT_ID"
v-if="form.database_id === EMPTY_OBJECT_ID"
color="var(--cl-warning-color)"
:icon="['fa', 'star']"
/>

7
frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx
View File

@@ -50,9 +50,6 @@ const useSpiderList = () => {
// action functions
const { deleteByIdConfirm } = actionFunctions;
const { allListSelectOptions: allProjectListSelectOptions } =
useProject(store);
// nav actions
const navActions = computed<ListActionGroup[]>(() => [
{
@@ -140,10 +137,6 @@ const useSpiderList = () => {
/>
);
},
hasFilter: true,
allowFilterSearch: true,
allowFilterItems: true,
filterItems: allProjectListSelectOptions.value,
},
{
key: 'git_id',

18
frontend/crawlab-ui/src/views/task/detail/actions/TaskDetailActionsData.vue
View File

@@ -14,14 +14,10 @@ const t = translate;
// store
const ns = 'task';
const store = useStore();
const { task: taskState } = store.state as RootStoreState;
const { allDict: allSpiderDict } = useSpider(store);
const { task: state } = store.state as RootStoreState;
// spider
const spider = computed(() =>
allSpiderDict.value.get(taskState.form.spider_id as string)
);
const spider = computed(() => state.form.spider);
// spider collection name
const colName = ref<string>();
@@ -34,16 +30,8 @@ watch(
}
);
// target
const target = () => colName.value;
// conditions
const conditions = () => [
{ key: '_tid', op: FILTER_OP_EQUAL, value: taskState.form._id },
];
// display all fields
const displayAllFields = ref<boolean>(taskState.dataDisplayAllFields);
const displayAllFields = ref<boolean>(state.dataDisplayAllFields);
const onDisplayAllFieldsChange = (val: boolean) => {
store.commit(`${ns}/setDataDisplayAllFields`, val);
};