From ec5bcf928717bfa8f560bf30ced550722f8cefef Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 12 May 2025 11:25:30 +0800 Subject: [PATCH] feat: introduce AutoProbe module and related interfaces - Added AutoProbe interfaces to define models for autoprobe functionality, including AutoProbe, AutoProbeTask, and AutoProbeFetchResult. - Created AutoProbeStore module for managing state, getters, mutations, and actions related to autoprobe. - Updated existing LLM interfaces to utilize LLMResponseUsage for token tracking. - Refactored store index to include autoprobe namespace and removed obsolete AiStore module. --- .../src/interfaces/models/autoprobe.d.ts | 74 +++++++++++++++++++ .../crawlab-ui/src/interfaces/models/llm.d.ts | 18 ++--- .../src/interfaces/store/index.d.ts | 8 +- .../src/interfaces/store/modules/ai.d.ts | 14 ---- .../interfaces/store/modules/autoprobe.d.ts | 14 ++++ .../crawlab-ui/src/store/modules/autoprobe.ts | 38 ++++++++++ .../views/autoprobe/list/useAutoProbeList.ts | 4 +- 7 files changed, 141 insertions(+), 29 deletions(-) create mode 100644 frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts delete mode 100644 frontend/crawlab-ui/src/interfaces/store/modules/ai.d.ts create mode 100644 frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts create mode 100644 frontend/crawlab-ui/src/store/modules/autoprobe.ts diff --git a/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts b/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts new file mode 100644 index 00000000..09f71546 --- /dev/null +++ b/frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts @@ -0,0 +1,74 @@ +export declare global { + interface AutoProbe extends BaseModel { + name?: string; + url?: string; + query?: string; + } + + type AutoProbeTaskStatus = 'pending' | 'running' | 'completed' | 'failed'; + + type SelectorType = 'css' | 'xpath' | 'regex'; + type ExtractType = 'text' | 'attribute' | 'html'; + type PaginationType = 'next' | 'load' | 'scroll'; + + interface FieldRule { + name: string; + selector_type: SelectorType; + selector: string; + extraction_type: ExtractType; + attribute_name?: string; + default_value?: string; + } + + interface ItemPattern { + fields?: FieldRule[]; + lists?: ListRule[]; + } + + interface ListRule { + name: string; + list_selector_type: SelectorType; + list_selector: string; + item_selector_type: SelectorType; + item_selector: string; + item_pattern: ItemPattern; + } + + interface Pagination { + type: PaginationType; + selector_type?: SelectorType; + selector?: string; + max_pages?: number; + start_page?: number; + } + + interface PagePattern { + name: string; + fields?: FieldRule[]; + lists?: ListRule[]; + pagination?: Pagination; + } + + interface PageData { + data?: Record; + list_data?: any[][]; + } + + interface AutoProbeTask extends BaseModel { + autoprobe_id: string; + url?: string; + query?: string; + status: AutoProbeTaskStatus; + page_pattern?: PagePattern; + page_data?: PageData; + provider_id?: string; + model?: string; + usage?: LLMResponseUsage; + } + + interface AutoProbeFetchResult extends BaseModel { + autoprobe_id: string; + url: string; + html?: string; + } +} diff --git a/frontend/crawlab-ui/src/interfaces/models/llm.d.ts b/frontend/crawlab-ui/src/interfaces/models/llm.d.ts index 5834dec9..73b229cf 100644 --- a/frontend/crawlab-ui/src/interfaces/models/llm.d.ts +++ b/frontend/crawlab-ui/src/interfaces/models/llm.d.ts @@ -1,4 +1,10 @@ export declare global { + interface LLMResponseUsage { + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + } + type LLMProviderType = | 'openai' | 'azure-openai' @@ -48,7 +54,7 @@ export declare global { metadata?: Record; status: ChatMessageStatus; error?: string; - usage?: ChatMessageUsage; + usage?: LLMResponseUsage; // Frontend UI-specific properties timestamp?: Date; @@ -68,18 +74,12 @@ export declare global { action_target?: string; action_status?: ChatMessageActionStatus; hidden?: boolean; - usage?: ChatMessageUsage; + usage?: LLMResponseUsage; // Frontend UI-specific properties isStreaming?: boolean; } - interface ChatMessageUsage { - prompt_tokens?: number; - completion_tokens?: number; - total_tokens?: number; - } - type ChatConversationStatus = 'active' | 'archived' | 'deleted'; interface ChatConversation extends BaseModel { @@ -133,6 +133,6 @@ export declare global { error?: string; hidden?: boolean; is_text_done?: boolean; - usage?: ChatMessageUsage; + usage?: LLMResponseUsage; } } diff --git a/frontend/crawlab-ui/src/interfaces/store/index.d.ts b/frontend/crawlab-ui/src/interfaces/store/index.d.ts index 4bad81d2..8cd4b0b2 100644 --- a/frontend/crawlab-ui/src/interfaces/store/index.d.ts +++ b/frontend/crawlab-ui/src/interfaces/store/index.d.ts @@ -29,15 +29,15 @@ export declare global { database: DatabaseStoreState; dependency: DependencyStoreState; environment: EnvironmentStoreState; - ai: AiStoreState; system: SystemStoreState; + autoprobe: AutoProbeStoreState; } type StoreGetter = ( state: S, getters?: GetterTree, rootState?: R, - rootGetters?: any + rootGetters?: any, ) => T; type StoreMutation = (state: S, payload: P) => void; @@ -45,7 +45,7 @@ export declare global { type StoreActionHandler = ( this: Store, ctx: ActionContext, - payload?: P + payload?: P, ) => T; interface StoreActionObject { @@ -175,7 +175,7 @@ export declare global { | 'dependency' | 'environment' | 'dataCollection' - | 'llmProvider'; + | 'autoprobe'; type StoreNamespace = ListStoreNamespace | 'layout' | 'common'; interface StoreContext { diff --git a/frontend/crawlab-ui/src/interfaces/store/modules/ai.d.ts b/frontend/crawlab-ui/src/interfaces/store/modules/ai.d.ts deleted file mode 100644 index 276faea4..00000000 --- a/frontend/crawlab-ui/src/interfaces/store/modules/ai.d.ts +++ /dev/null @@ -1,14 +0,0 @@ -type AiStoreModule = BaseModule< - AiStoreState, - AiStoreGetters, - AiStoreMutations, - AiStoreActions ->; - -interface AiStoreState extends BaseStoreState {} - -type AiStoreGetters = BaseStoreGetters; - -interface AiStoreMutations extends BaseStoreMutations {} - -interface AiStoreActions extends BaseStoreActions {} diff --git a/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts b/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts new file mode 100644 index 00000000..5a5901b0 --- /dev/null +++ b/frontend/crawlab-ui/src/interfaces/store/modules/autoprobe.d.ts @@ -0,0 +1,14 @@ +type AutoProbeStoreModule = BaseModule< + AutoProbeStoreState, + AutoProbeStoreGetters, + AutoProbeStoreMutations, + AutoProbeStoreActions +>; + +interface AutoProbeStoreState extends BaseStoreState {} + +type AutoProbeStoreGetters = BaseStoreGetters; + +interface AutoProbeStoreMutations extends BaseStoreMutations {} + +interface AutoProbeStoreActions extends BaseStoreActions {} diff --git a/frontend/crawlab-ui/src/store/modules/autoprobe.ts b/frontend/crawlab-ui/src/store/modules/autoprobe.ts new file mode 100644 index 00000000..d53f6a91 --- /dev/null +++ b/frontend/crawlab-ui/src/store/modules/autoprobe.ts @@ -0,0 +1,38 @@ +import { + getDefaultStoreActions, + getDefaultStoreGetters, + getDefaultStoreMutations, + getDefaultStoreState, +} from '@/utils/store'; +import { TAB_NAME_OVERVIEW } from '@/constants/tab'; +import { translate } from '@/utils/i18n'; + +// i18n +const t = translate; + +const state = { + ...getDefaultStoreState('autoprobe'), + tabs: [ + { id: TAB_NAME_OVERVIEW, title: t('common.tabs.overview') }, + ], +} as AutoProbeStoreState; + +const getters = { + ...getDefaultStoreGetters(), +} as AutoProbeStoreGetters; + +const mutations = { + ...getDefaultStoreMutations(), +} as AutoProbeStoreMutations; + +const actions = { + ...getDefaultStoreActions('/projects'), +} as AutoProbeStoreActions; + +export default { + namespaced: true, + state, + getters, + mutations, + actions, +} as AutoProbeStoreModule; diff --git a/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.ts b/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.ts index 266c6345..49aa9fe3 100644 --- a/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.ts +++ b/frontend/crawlab-ui/src/views/autoprobe/list/useAutoProbeList.ts @@ -2,11 +2,11 @@ import { useStore } from 'vuex'; import { useList } from '@/layouts'; const useAutoProbeList = () => { - const ns: ListStoreNamespace = 'extract'; + const ns: ListStoreNamespace = 'autoprobe'; const store = useStore(); return { - ...useList(ns), + ...useList(ns, store), }; };