From c06a9c605202dae3eeb71db2c3c29aea0968effe Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 16 Jun 2025 22:54:33 +0800 Subject: [PATCH] feat: update AutoProbe service and interfaces to support AutoProbeV2 --- .../components/core/autoprobe/useAutoProbe.ts | 4 +- .../src/interfaces/models/autoprobe.d.ts | 16 +++ .../interfaces/store/modules/autoprobe.d.ts | 8 +- .../services/autoprobe/autoprobeService.ts | 4 +- .../crawlab-ui/src/store/modules/autoprobe.ts | 46 +++++-- .../tabs/AutoProbeDetailTabPatterns.vue | 114 ++++++------------ .../views/autoprobe/list/useAutoProbeList.tsx | 18 +-- 7 files changed, 105 insertions(+), 105 deletions(-) diff --git a/frontend/crawlab-ui/src/components/core/autoprobe/useAutoProbe.ts b/frontend/crawlab-ui/src/components/core/autoprobe/useAutoProbe.ts index 55bfc714..8fdabd82 100644 --- a/frontend/crawlab-ui/src/components/core/autoprobe/useAutoProbe.ts +++ b/frontend/crawlab-ui/src/components/core/autoprobe/useAutoProbe.ts @@ -5,7 +5,7 @@ import useAutoProbeService from '@/services/autoprobe/autoprobeService'; import { getDefaultFormComponentData } from '@/utils/form'; // form component data -const formComponentData = getDefaultFormComponentData(); +const formComponentData = getDefaultFormComponentData(); const useAutoProbe = (store: Store) => { // store @@ -20,7 +20,7 @@ const useAutoProbe = (store: Store) => { }; return { - ...useForm( + ...useForm( 'autoprobe', store, useAutoProbeService(store), diff --git a/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts b/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts index 01b67ed1..7868b80d 100644 --- a/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts +++ b/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts @@ -13,6 +13,22 @@ export declare global { viewport?: PageViewPort; } + // V2 AutoProbe interface that matches backend AutoProbeV2 + interface AutoProbeV2 extends BaseModel { + name?: string; + description?: string; + url?: string; + query?: string; + last_task_id?: string; + last_task_status?: AutoProbeTaskStatus; + last_task_error?: string; + default_task_id?: string; + run_on_create?: boolean; + page_pattern?: PagePatternV2; + page_data?: PageData; + viewport?: PageViewPort; + } + // Hierarchical pattern structure for V2 interface PatternV2 extends BaseModel { name: string; diff --git a/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts b/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts index 9e9aa4b3..b318ab9d 100644 --- a/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts +++ b/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts @@ -5,21 +5,21 @@ type AutoProbeStoreModule = BaseModule< AutoProbeStoreActions >; -interface AutoProbeStoreState extends BaseStoreState { +interface AutoProbeStoreState extends BaseStoreState { pagePattern?: PagePatternV2; pagePatternData?: PatternDataV2[]; } -type AutoProbeStoreGetters = BaseStoreGetters; +type AutoProbeStoreGetters = BaseStoreGetters; -interface AutoProbeStoreMutations extends BaseStoreMutations { +interface AutoProbeStoreMutations extends BaseStoreMutations { setPagePattern: StoreMutation; resetPagePattern: StoreMutation; setPagePatternData: StoreMutation; resetPagePatternData: StoreMutation; } -interface AutoProbeStoreActions extends BaseStoreActions { +interface AutoProbeStoreActions extends BaseStoreActions { runTask: StoreAction; cancelTask: StoreAction; getPagePattern: StoreAction; diff --git a/frontend/crawlab-ui/src/services/autoprobe/autoprobeService.ts b/frontend/crawlab-ui/src/services/autoprobe/autoprobeService.ts index 48ce4c38..feaea4c9 100644 --- a/frontend/crawlab-ui/src/services/autoprobe/autoprobeService.ts +++ b/frontend/crawlab-ui/src/services/autoprobe/autoprobeService.ts @@ -3,11 +3,11 @@ import { getDefaultService } from '@/utils'; const useAutoProbeService = ( store: Store -): Services => { +): Services => { const ns: ListStoreNamespace = 'autoprobe'; return { - ...getDefaultService(ns, store), + ...getDefaultService(ns, store), }; }; diff --git a/frontend/crawlab-ui/src/store/modules/autoprobe.ts b/frontend/crawlab-ui/src/store/modules/autoprobe.ts index 9629a8fa..f8c8f75c 100644 --- a/frontend/crawlab-ui/src/store/modules/autoprobe.ts +++ b/frontend/crawlab-ui/src/store/modules/autoprobe.ts @@ -16,10 +16,10 @@ import { getViewPortOptions } from '@/utils'; // i18n const t = translate; -const { post } = useRequest(); +const { get, post } = useRequest(); const state = { - ...getDefaultStoreState('autoprobe'), + ...getDefaultStoreState('autoprobe'), newFormFn: () => { return { run_on_create: true, @@ -37,11 +37,11 @@ const state = { } as AutoProbeStoreState; const getters = { - ...getDefaultStoreGetters(), + ...getDefaultStoreGetters(), } as AutoProbeStoreGetters; const mutations = { - ...getDefaultStoreMutations(), + ...getDefaultStoreMutations(), setPagePattern(state: AutoProbeStoreState, pagePattern: PagePatternV2) { state.pagePattern = pagePattern; }, @@ -62,7 +62,7 @@ const mutations = { const endpoint = '/ai/autoprobes'; const actions = { - ...getDefaultStoreActions(endpoint), + ...getDefaultStoreActions(endpoint), runTask: async ( _: StoreActionContext, { id }: { id: string } @@ -76,18 +76,46 @@ const actions = { await post(`${endpoint}/tasks/${id}/cancel`); }, getPagePattern: async ( - { commit }: StoreActionContext, + { commit, state }: StoreActionContext, { id }: { id: string } ) => { - const res = await post(`${endpoint}/${id}/pattern`); + const res = await get(`${endpoint}/${id}/pattern`); commit('setPagePattern', res.data); + + // Also update the form data so the component can access it + if (state.form) { + commit('setForm', { + ...state.form, + page_pattern: res.data + }); + } }, getPagePatternData: async ( - { commit }: StoreActionContext, + { commit, state }: StoreActionContext, { id }: { id: string } ) => { - const res = await post(`${endpoint}/${id}/pattern/data`); + const res = await get(`${endpoint}/${id}/pattern/results`); commit('setPagePatternData', res.data); + + // Transform PatternDataV2[] array into structured page data object + const structuredData: Record = {}; + if (Array.isArray(res.data)) { + res.data.forEach((patternData: any) => { + // For now, just use a simple mapping - we might need to enhance this later + // based on how the pattern hierarchy should map to data + if (patternData.pattern_id && patternData.data !== undefined) { + structuredData[patternData.pattern_id] = patternData.data; + } + }); + } + + // Also update the form data so the component can access it + if (state.form) { + commit('setForm', { + ...state.form, + page_data: structuredData + }); + } }, } as AutoProbeStoreActions; diff --git a/frontend/crawlab-ui/src/views/autoprobe/detail/tabs/AutoProbeDetailTabPatterns.vue b/frontend/crawlab-ui/src/views/autoprobe/detail/tabs/AutoProbeDetailTabPatterns.vue index 75feda7e..290382de 100644 --- a/frontend/crawlab-ui/src/views/autoprobe/detail/tabs/AutoProbeDetailTabPatterns.vue +++ b/frontend/crawlab-ui/src/views/autoprobe/detail/tabs/AutoProbeDetailTabPatterns.vue @@ -16,10 +16,8 @@ const { autoprobe: state } = store.state as RootStoreState; const { activeId } = useAutoProbeDetail(); // form data -const form = computed(() => state.form); -const pageFields = computed(() => form.value?.page_pattern?.fields); -const pageLists = computed(() => form.value?.page_pattern?.lists); -const pagePagination = computed(() => form.value?.page_pattern?.pagination); +const form = computed(() => state.form); +const pagePattern = computed(() => form.value?.page_pattern as PagePatternV2); const pageData = computed(() => form.value?.page_data || {}); const pageNavItemId = 'page'; @@ -27,7 +25,7 @@ const pageNavItemId = 'page'; const resultsDataFields = computed(() => { const rootDataFields: AutoProbeResults = { data: pageData.value, - fields: computedTreeItems.value[0].children?.filter( + fields: computedTreeItems.value[0]?.children?.filter( item => item.type !== 'pagination' ), }; @@ -42,8 +40,11 @@ const resultsDataFields = computed(() => { return rootDataFields; } else if (item.level === 1) { if (item.type === 'list') { + // For V2 patterns, use pattern ID to get the data + const pattern = item.rule as PatternV2; + const patternId = pattern._id || pattern.name; return { - data: pageData.value[item.name!], + data: pageData.value[patternId], fields: item.children, } as AutoProbeResults; } @@ -56,8 +57,11 @@ const resultsDataFields = computed(() => { while (currentItem.parent) { const parent = currentItem.parent; if (parent.level === 1 && parent.type === 'list') { + // For V2 patterns, use pattern ID to get the data + const parentPattern = parent.rule as PatternV2; + const parentPatternId = parentPattern._id || parentPattern.name; return { - data: pageData.value[parent.name!], + data: pageData.value[parentPatternId], fields: parent.children, activeField: currentItem, } as AutoProbeResults; @@ -75,8 +79,9 @@ const normalizeItem = (item: AutoProbeNavItem) => { const label = item.label ?? `${item.name} (${item.children?.length || 0})`; let icon: Icon; if (item.type === 'field') { - const field = item.rule as FieldRule; - icon = getIconByExtractType(field.extraction_type); + // For V2 patterns, extraction_type is directly on the pattern object + const pattern = item.rule as PatternV2; + icon = getIconByExtractType(pattern.extraction_type); } else { icon = getIconByItemType(item.type); } @@ -87,47 +92,30 @@ const normalizeItem = (item: AutoProbeNavItem) => { } as AutoProbeNavItem; }; -// Helper function to recursively process list items -const processListItem = ( - list: ListRule, +// Helper function to recursively process V2 patterns +const processPatternV2 = ( + pattern: PatternV2, parent?: AutoProbeNavItem, level: number = 1 ): AutoProbeNavItem => { - const listItem: AutoProbeNavItem = { - id: list.name, - name: list.name, - type: 'list', - rule: list, + const navItem: AutoProbeNavItem = { + id: pattern._id || pattern.name, + name: pattern.name, + type: pattern.type as AutoProbeItemType, + rule: pattern as any, // PatternV2 structure is compatible, just cast for type compatibility children: [], parent, level, }; - // Add fields directly if they exist - if (list.item_pattern?.fields && list.item_pattern.fields.length > 0) { - list.item_pattern.fields.forEach((field: FieldRule) => { - listItem.children!.push( - normalizeItem({ - id: `${list.name}-${field.name}`, - label: field.name, - name: field.name, - type: 'field', - rule: field, - parent: listItem, - level: level + 1, - }) - ); + // Recursively process child patterns + if (pattern.children && pattern.children.length > 0) { + pattern.children.forEach((childPattern: PatternV2) => { + navItem.children!.push(processPatternV2(childPattern, navItem, level + 1)); }); } - // Recursively process nested lists if they exist - if (list.item_pattern?.lists && list.item_pattern.lists.length > 0) { - list.item_pattern.lists.forEach((nestedList: ListRule) => { - listItem.children!.push(processListItem(nestedList, listItem, level + 1)); - }); - } - - return normalizeItem(listItem); + return normalizeItem(navItem); }; // items @@ -144,55 +132,23 @@ const detailNavItem = computed(() => { } }); const computedTreeItems = computed(() => { - if (!form.value?.page_pattern) return []; + if (!pagePattern.value) return []; const rootItem: AutoProbeNavItem = { id: pageNavItemId, - name: form.value.page_pattern.name, + name: pagePattern.value.name, type: 'page_pattern', children: [], level: 0, }; - // Add fields directly if they exist - if (pageFields.value) { - pageFields.value.forEach(field => { - rootItem.children!.push( - normalizeItem({ - id: field.name, - label: field.name, - name: field.name, - type: 'field', - rule: field, - parent: rootItem, - level: 1, - }) - ); + // Process V2 pattern children + if (pagePattern.value.children && pagePattern.value.children.length > 0) { + pagePattern.value.children.forEach(pattern => { + rootItem.children!.push(processPatternV2(pattern, rootItem, 1)); }); } - // Add lists directly if they exist - if (pageLists.value) { - pageLists.value.forEach(list => { - rootItem.children!.push(processListItem(list, rootItem, 1)); - }); - } - - // Add pagination if it exists - if (pagePagination.value) { - rootItem.children!.push( - normalizeItem({ - id: 'pagination', - label: t('components.autoprobe.navItems.pagination'), - name: t('components.autoprobe.navItems.pagination'), - type: 'pagination', - rule: pagePagination.value, - parent: rootItem, - level: 1, - }) - ); - } - return [normalizeItem(rootItem)]; }); const treeItems = ref([]); @@ -235,8 +191,8 @@ const onSizeChange = (size: number) => { const getData = debounce(async () => { await Promise.all([ - store.dispatch(`${ns}/getPagePattern`), - store.dispatch(`${ns}/getPagePatternData`), + store.dispatch(`${ns}/getPagePattern`, { id: activeId.value }), + store.dispatch(`${ns}/getPagePatternData`, { id: activeId.value }), ]); }); watch(activeId, getData); diff --git a/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.tsx b/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.tsx index dd5f45ee..545da4f6 100644 --- a/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.tsx +++ b/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.tsx @@ -34,7 +34,7 @@ const useAutoProbeList = () => { const store = useStore(); const { commit } = store; - const { actionFunctions } = useList(ns, store); + const { actionFunctions } = useList(ns, store); const { getList, deleteByIdConfirm } = actionFunctions; // nav actions @@ -79,7 +79,7 @@ const useAutoProbeList = () => { ]); // table columns - const tableColumns = computed>( + const tableColumns = computed>( () => [ { @@ -88,7 +88,7 @@ const useAutoProbeList = () => { label: t('views.autoprobe.table.columns.name'), icon: ['fa', 'font'], width: '150', - value: (row: AutoProbe) => ( + value: (row: AutoProbeV2) => ( ), hasSort: true, @@ -101,7 +101,7 @@ const useAutoProbeList = () => { icon: ['fa', 'at'], width: 'auto', minWidth: '200', - value: (row: AutoProbe) => ( + value: (row: AutoProbeV2) => ( ), hasFilter: true, @@ -112,7 +112,7 @@ const useAutoProbeList = () => { label: t('views.autoprobe.table.columns.lastTask'), icon: ['fa', 'heartbeat'], width: '120', - value: (row: AutoProbe) => { + value: (row: AutoProbeV2) => { const status = row.last_task_status; const error = row.last_task_error; if (!status) return; @@ -188,14 +188,14 @@ const useAutoProbeList = () => { ], disableTransfer: true, }, - ] as TableColumns + ] as TableColumns ); - const rowKey = (row: AutoProbe) => { + const rowKey = (row: AutoProbeV2) => { return JSON.stringify([ row._id, row.url, - row.last_task?.status, + row.last_task_status, row.page_pattern, ]); }; @@ -203,7 +203,7 @@ const useAutoProbeList = () => { setupAutoUpdate(getList); return { - ...useList(ns, store), + ...useList(ns, store), navActions, tableColumns, rowKey,