feat: update AutoProbe service and interfaces to support AutoProbeV2

2026-01-21 17:21:09 +01:00 · 2025-06-16 22:54:33 +08:00
parent 24452bff23
commit c06a9c6052
7 changed files with 105 additions and 105 deletions
--- a/frontend/crawlab-ui/src/components/core/autoprobe/useAutoProbe.ts
+++ b/frontend/crawlab-ui/src/components/core/autoprobe/useAutoProbe.ts
@@ -5,7 +5,7 @@ import useAutoProbeService from '@/services/autoprobe/autoprobeService';
 import { getDefaultFormComponentData } from '@/utils/form';

 // form component data
-const formComponentData = getDefaultFormComponentData<AutoProbe>();
+const formComponentData = getDefaultFormComponentData<AutoProbeV2>();

 const useAutoProbe = (store: Store<RootStoreState>) => {
  // store
@@ -20,7 +20,7 @@ const useAutoProbe = (store: Store<RootStoreState>) => {
  };

  return {
-    ...useForm<AutoProbe>(
+    ...useForm<AutoProbeV2>(
      'autoprobe',
      store,
      useAutoProbeService(store),
--- a/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts
+++ b/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts
@@ -13,6 +13,22 @@ export declare global {
    viewport?: PageViewPort;
  }

+  // V2 AutoProbe interface that matches backend AutoProbeV2
+  interface AutoProbeV2 extends BaseModel {
+    name?: string;
+    description?: string;
+    url?: string;
+    query?: string;
+    last_task_id?: string;
+    last_task_status?: AutoProbeTaskStatus;
+    last_task_error?: string;
+    default_task_id?: string;
+    run_on_create?: boolean;
+    page_pattern?: PagePatternV2;
+    page_data?: PageData;
+    viewport?: PageViewPort;
+  }
+
  // Hierarchical pattern structure for V2
  interface PatternV2 extends BaseModel {
    name: string;
--- a/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts
+++ b/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts
@@ -5,21 +5,21 @@ type AutoProbeStoreModule = BaseModule<
  AutoProbeStoreActions
 >;

-interface AutoProbeStoreState extends BaseStoreState<AutoProbe> {
+interface AutoProbeStoreState extends BaseStoreState<AutoProbeV2> {
  pagePattern?: PagePatternV2;
  pagePatternData?: PatternDataV2[];
 }

-type AutoProbeStoreGetters = BaseStoreGetters<AutoProbe>;
+type AutoProbeStoreGetters = BaseStoreGetters<AutoProbeV2>;

-interface AutoProbeStoreMutations extends BaseStoreMutations<AutoProbe> {
+interface AutoProbeStoreMutations extends BaseStoreMutations<AutoProbeV2> {
  setPagePattern: StoreMutation<AutoProbeStoreState, PagePatternV2>;
  resetPagePattern: StoreMutation<AutoProbeStoreState>;
  setPagePatternData: StoreMutation<AutoProbeStoreState, PatternDataV2[]>;
  resetPagePatternData: StoreMutation<AutoProbeStoreState>;
 }

-interface AutoProbeStoreActions extends BaseStoreActions<AutoProbe> {
+interface AutoProbeStoreActions extends BaseStoreActions<AutoProbeV2> {
  runTask: StoreAction<AutoProbeStoreState, { id: string }>;
  cancelTask: StoreAction<AutoProbeStoreState, { id: string }>;
  getPagePattern: StoreAction<AutoProbeStoreState, { id: string }>;
--- a/frontend/crawlab-ui/src/services/autoprobe/autoprobeService.ts
+++ b/frontend/crawlab-ui/src/services/autoprobe/autoprobeService.ts
@@ -3,11 +3,11 @@ import { getDefaultService } from '@/utils';

 const useAutoProbeService = (
  store: Store<RootStoreState>
-): Services<AutoProbe> => {
+): Services<AutoProbeV2> => {
  const ns: ListStoreNamespace = 'autoprobe';

  return {
-    ...getDefaultService<AutoProbe>(ns, store),
+    ...getDefaultService<AutoProbeV2>(ns, store),
  };
 };

--- a/frontend/crawlab-ui/src/store/modules/autoprobe.ts
+++ b/frontend/crawlab-ui/src/store/modules/autoprobe.ts
@@ -16,10 +16,10 @@ import { getViewPortOptions } from '@/utils';
 // i18n
 const t = translate;

-const { post } = useRequest();
+const { get, post } = useRequest();

 const state = {
-  ...getDefaultStoreState<AutoProbe>('autoprobe'),
+  ...getDefaultStoreState<AutoProbeV2>('autoprobe'),
  newFormFn: () => {
    return {
      run_on_create: true,
@@ -37,11 +37,11 @@ const state = {
 } as AutoProbeStoreState;

 const getters = {
-  ...getDefaultStoreGetters<AutoProbe>(),
+  ...getDefaultStoreGetters<AutoProbeV2>(),
 } as AutoProbeStoreGetters;

 const mutations = {
-  ...getDefaultStoreMutations<AutoProbe>(),
+  ...getDefaultStoreMutations<AutoProbeV2>(),
  setPagePattern(state: AutoProbeStoreState, pagePattern: PagePatternV2) {
    state.pagePattern = pagePattern;
  },
@@ -62,7 +62,7 @@ const mutations = {
 const endpoint = '/ai/autoprobes';

 const actions = {
-  ...getDefaultStoreActions<AutoProbe>(endpoint),
+  ...getDefaultStoreActions<AutoProbeV2>(endpoint),
  runTask: async (
    _: StoreActionContext<AutoProbeStoreState>,
    { id }: { id: string }
@@ -76,18 +76,46 @@ const actions = {
    await post(`${endpoint}/tasks/${id}/cancel`);
  },
  getPagePattern: async (
-    { commit }: StoreActionContext<AutoProbeStoreState>,
+    { commit, state }: StoreActionContext<AutoProbeStoreState>,
    { id }: { id: string }
  ) => {
-    const res = await post(`${endpoint}/${id}/pattern`);
+    const res = await get(`${endpoint}/${id}/pattern`);
    commit('setPagePattern', res.data);
+    
+    // Also update the form data so the component can access it
+    if (state.form) {
+      commit('setForm', {
+        ...state.form,
+        page_pattern: res.data
+      });
+    }
  },
  getPagePatternData: async (
-    { commit }: StoreActionContext<AutoProbeStoreState>,
+    { commit, state }: StoreActionContext<AutoProbeStoreState>,
    { id }: { id: string }
  ) => {
-    const res = await post(`${endpoint}/${id}/pattern/data`);
+    const res = await get(`${endpoint}/${id}/pattern/results`);
    commit('setPagePatternData', res.data);
+    
+    // Transform PatternDataV2[] array into structured page data object
+    const structuredData: Record<string, any> = {};
+    if (Array.isArray(res.data)) {
+      res.data.forEach((patternData: any) => {
+        // For now, just use a simple mapping - we might need to enhance this later
+        // based on how the pattern hierarchy should map to data
+        if (patternData.pattern_id && patternData.data !== undefined) {
+          structuredData[patternData.pattern_id] = patternData.data;
+        }
+      });
+    }
+    
+    // Also update the form data so the component can access it
+    if (state.form) {
+      commit('setForm', {
+        ...state.form,
+        page_data: structuredData
+      });
+    }
  },
 } as AutoProbeStoreActions;

--- a/frontend/crawlab-ui/src/views/autoprobe/detail/tabs/AutoProbeDetailTabPatterns.vue
+++ b/frontend/crawlab-ui/src/views/autoprobe/detail/tabs/AutoProbeDetailTabPatterns.vue
@@ -16,10 +16,8 @@ const { autoprobe: state } = store.state as RootStoreState;
 const { activeId } = useAutoProbeDetail();

 // form data
-const form = computed<AutoProbe>(() => state.form);
-const pageFields = computed(() => form.value?.page_pattern?.fields);
-const pageLists = computed(() => form.value?.page_pattern?.lists);
-const pagePagination = computed(() => form.value?.page_pattern?.pagination);
+const form = computed<AutoProbeV2>(() => state.form);
+const pagePattern = computed(() => form.value?.page_pattern as PagePatternV2);
 const pageData = computed<PageData>(() => form.value?.page_data || {});
 const pageNavItemId = 'page';

@@ -27,7 +25,7 @@ const pageNavItemId = 'page';
 const resultsDataFields = computed<AutoProbeResults>(() => {
  const rootDataFields: AutoProbeResults = {
    data: pageData.value,
-    fields: computedTreeItems.value[0].children?.filter(
+    fields: computedTreeItems.value[0]?.children?.filter(
      item => item.type !== 'pagination'
    ),
  };
@@ -42,8 +40,11 @@ const resultsDataFields = computed<AutoProbeResults>(() => {
    return rootDataFields;
  } else if (item.level === 1) {
    if (item.type === 'list') {
+      // For V2 patterns, use pattern ID to get the data
+      const pattern = item.rule as PatternV2;
+      const patternId = pattern._id || pattern.name;
      return {
-        data: pageData.value[item.name!],
+        data: pageData.value[patternId],
        fields: item.children,
      } as AutoProbeResults;
    }
@@ -56,8 +57,11 @@ const resultsDataFields = computed<AutoProbeResults>(() => {
    while (currentItem.parent) {
      const parent = currentItem.parent;
      if (parent.level === 1 && parent.type === 'list') {
+        // For V2 patterns, use pattern ID to get the data
+        const parentPattern = parent.rule as PatternV2;
+        const parentPatternId = parentPattern._id || parentPattern.name;
        return {
-          data: pageData.value[parent.name!],
+          data: pageData.value[parentPatternId],
          fields: parent.children,
          activeField: currentItem,
        } as AutoProbeResults;
@@ -75,8 +79,9 @@ const normalizeItem = (item: AutoProbeNavItem) => {
  const label = item.label ?? `${item.name} (${item.children?.length || 0})`;
  let icon: Icon;
  if (item.type === 'field') {
-    const field = item.rule as FieldRule;
-    icon = getIconByExtractType(field.extraction_type);
+    // For V2 patterns, extraction_type is directly on the pattern object
+    const pattern = item.rule as PatternV2;
+    icon = getIconByExtractType(pattern.extraction_type);
  } else {
    icon = getIconByItemType(item.type);
  }
@@ -87,47 +92,30 @@ const normalizeItem = (item: AutoProbeNavItem) => {
  } as AutoProbeNavItem;
 };

-// Helper function to recursively process list items
-const processListItem = (
-  list: ListRule,
+// Helper function to recursively process V2 patterns
+const processPatternV2 = (
+  pattern: PatternV2,
  parent?: AutoProbeNavItem,
  level: number = 1
 ): AutoProbeNavItem => {
-  const listItem: AutoProbeNavItem = {
-    id: list.name,
-    name: list.name,
-    type: 'list',
-    rule: list,
+  const navItem: AutoProbeNavItem = {
+    id: pattern._id || pattern.name,
+    name: pattern.name,
+    type: pattern.type as AutoProbeItemType,
+    rule: pattern as any, // PatternV2 structure is compatible, just cast for type compatibility
    children: [],
    parent,
    level,
  };

-  // Add fields directly if they exist
-  if (list.item_pattern?.fields && list.item_pattern.fields.length > 0) {
-    list.item_pattern.fields.forEach((field: FieldRule) => {
-      listItem.children!.push(
-        normalizeItem({
-          id: `${list.name}-${field.name}`,
-          label: field.name,
-          name: field.name,
-          type: 'field',
-          rule: field,
-          parent: listItem,
-          level: level + 1,
-        })
-      );
+  // Recursively process child patterns
+  if (pattern.children && pattern.children.length > 0) {
+    pattern.children.forEach((childPattern: PatternV2) => {
+      navItem.children!.push(processPatternV2(childPattern, navItem, level + 1));
    });
  }

-  // Recursively process nested lists if they exist
-  if (list.item_pattern?.lists && list.item_pattern.lists.length > 0) {
-    list.item_pattern.lists.forEach((nestedList: ListRule) => {
-      listItem.children!.push(processListItem(nestedList, listItem, level + 1));
-    });
-  }
-
-  return normalizeItem(listItem);
+  return normalizeItem(navItem);
 };

 // items
@@ -144,55 +132,23 @@ const detailNavItem = computed<AutoProbeNavItem | undefined>(() => {
  }
 });
 const computedTreeItems = computed<AutoProbeNavItem[]>(() => {
-  if (!form.value?.page_pattern) return [];
+  if (!pagePattern.value) return [];

  const rootItem: AutoProbeNavItem = {
    id: pageNavItemId,
-    name: form.value.page_pattern.name,
+    name: pagePattern.value.name,
    type: 'page_pattern',
    children: [],
    level: 0,
  };

-  // Add fields directly if they exist
-  if (pageFields.value) {
-    pageFields.value.forEach(field => {
-      rootItem.children!.push(
-        normalizeItem({
-          id: field.name,
-          label: field.name,
-          name: field.name,
-          type: 'field',
-          rule: field,
-          parent: rootItem,
-          level: 1,
-        })
-      );
+  // Process V2 pattern children
+  if (pagePattern.value.children && pagePattern.value.children.length > 0) {
+    pagePattern.value.children.forEach(pattern => {
+      rootItem.children!.push(processPatternV2(pattern, rootItem, 1));
    });
  }

-  // Add lists directly if they exist
-  if (pageLists.value) {
-    pageLists.value.forEach(list => {
-      rootItem.children!.push(processListItem(list, rootItem, 1));
-    });
-  }
-
-  // Add pagination if it exists
-  if (pagePagination.value) {
-    rootItem.children!.push(
-      normalizeItem({
-        id: 'pagination',
-        label: t('components.autoprobe.navItems.pagination'),
-        name: t('components.autoprobe.navItems.pagination'),
-        type: 'pagination',
-        rule: pagePagination.value,
-        parent: rootItem,
-        level: 1,
-      })
-    );
-  }
-
  return [normalizeItem(rootItem)];
 });
 const treeItems = ref<AutoProbeNavItem[]>([]);
@@ -235,8 +191,8 @@ const onSizeChange = (size: number) => {

 const getData = debounce(async () => {
  await Promise.all([
-    store.dispatch(`${ns}/getPagePattern`),
-    store.dispatch(`${ns}/getPagePatternData`),
+    store.dispatch(`${ns}/getPagePattern`, { id: activeId.value }),
+    store.dispatch(`${ns}/getPagePatternData`, { id: activeId.value }),
  ]);
 });
 watch(activeId, getData);
--- a/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.tsx
+++ b/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.tsx
@@ -34,7 +34,7 @@ const useAutoProbeList = () => {
  const store = useStore();
  const { commit } = store;

-  const { actionFunctions } = useList<AutoProbe>(ns, store);
+  const { actionFunctions } = useList<AutoProbeV2>(ns, store);
  const { getList, deleteByIdConfirm } = actionFunctions;

  // nav actions
@@ -79,7 +79,7 @@ const useAutoProbeList = () => {
  ]);

  // table columns
-  const tableColumns = computed<TableColumns<AutoProbe>>(
+  const tableColumns = computed<TableColumns<AutoProbeV2>>(
    () =>
      [
        {
@@ -88,7 +88,7 @@ const useAutoProbeList = () => {
          label: t('views.autoprobe.table.columns.name'),
          icon: ['fa', 'font'],
          width: '150',
-          value: (row: AutoProbe) => (
+          value: (row: AutoProbeV2) => (
            <ClNavLink path={`/autoprobes/${row._id}`} label={row.name} />
          ),
          hasSort: true,
@@ -101,7 +101,7 @@ const useAutoProbeList = () => {
          icon: ['fa', 'at'],
          width: 'auto',
          minWidth: '200',
-          value: (row: AutoProbe) => (
+          value: (row: AutoProbeV2) => (
            <ClNavLink path={row.url} label={row.url} external />
          ),
          hasFilter: true,
@@ -112,7 +112,7 @@ const useAutoProbeList = () => {
          label: t('views.autoprobe.table.columns.lastTask'),
          icon: ['fa', 'heartbeat'],
          width: '120',
-          value: (row: AutoProbe) => {
+          value: (row: AutoProbeV2) => {
            const status = row.last_task_status;
            const error = row.last_task_error;
            if (!status) return;
@@ -188,14 +188,14 @@ const useAutoProbeList = () => {
          ],
          disableTransfer: true,
        },
-      ] as TableColumns<AutoProbe>
+      ] as TableColumns<AutoProbeV2>
  );

-  const rowKey = (row: AutoProbe) => {
+  const rowKey = (row: AutoProbeV2) => {
    return JSON.stringify([
      row._id,
      row.url,
-      row.last_task?.status,
+      row.last_task_status,
      row.page_pattern,
    ]);
  };
@@ -203,7 +203,7 @@ const useAutoProbeList = () => {
  setupAutoUpdate(getList);

  return {
-    ...useList<AutoProbe>(ns, store),
+    ...useList<AutoProbeV2>(ns, store),
    navActions,
    tableColumns,
    rowKey,