git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: update AutoProbe components and enhance structure

Browse Source 
- Removed obsolete AutoProbeFieldDetail, AutoProbeListDetail, AutoProbePagePatternDetail, and AutoProbePaginationDetail components to streamline the codebase.
- Introduced new components: AutoProbeItemDetail and AutoProbePagePatternsSidebar for improved data management and user interaction.
- Updated index.ts to reflect new component structure and exports.
- Enhanced internationalization support with new translations for preview and extraction types.
- Refactored AutoProbeDetailTabPatterns to utilize new components and improve data handling.
This commit is contained in:
Marvin Zhang
2025-05-15 18:04:27 +08:00
parent c654f03890
commit f3d6a56315
23 changed files with 1015 additions and 1475 deletions
Download Patch File Download Diff File Expand all files Collapse all files

270
frontend/crawlab-ui/src/components/core/autoprobe/AutoProbeFieldDetail.vue
View File

@@ -1,270 +0,0 @@
<script setup lang="ts">
import { computed } from 'vue';
import { useStore } from 'vuex';
import { translate } from '@/utils';
// i18n
const t = translate;
// props
const props = defineProps<{
field: AutoProbeNavItem;
pageData?: any;
}>();
// store
const store = useStore();
const { autoprobe: state } = store.state as RootStoreState;
const form = computed<AutoProbe>(() => state.form);
// Find the actual field data from the form based on the nav item
const fieldData = computed<FieldRule | undefined>(() => {
if (!form.value?.page_pattern) return undefined;
// If it's a direct field of the page pattern
if (props.field.id.indexOf('-') === -1) {
return form.value.page_pattern.fields?.find(f => f.name === props.field.label);
}
// If it's a field within a list, parse the ID
const [listName, fieldName] = props.field.id.split('-');
// Find the list first
const findList = (lists: ListRule[] | undefined): ListRule | undefined => {
if (!lists) return undefined;
for (const list of lists) {
if (list.name === listName) return list;
const nestedResult = findList(list.item_pattern?.lists);
if (nestedResult) return nestedResult;
}
return undefined;
};
const list = findList(form.value.page_pattern.lists);
if (!list || !list.item_pattern?.fields) return undefined;
return list.item_pattern.fields.find(f => f.name === fieldName);
});
// Display value for selector - if empty, it points to itself (self)
const displaySelector = computed(() => {
if (!fieldData.value?.selector) return t('components.autoprobe.field.self') || 'self';
return fieldData.value.selector;
});
// Determine if we have tabular data (array of objects)
const isTabularData = computed(() => {
if (!props.pageData) return false;
// Check if it's an array with objects inside
if (Array.isArray(props.pageData) && props.pageData.length > 0 &&
typeof props.pageData[0] === 'object' && props.pageData[0] !== null) {
return true;
}
return false;
});
// Get table data for tabular display
const tableData = computed(() => {
if (!props.pageData || !isTabularData.value) return [];
return props.pageData;
});
// Get table columns
const tableColumns = computed(() => {
if (!tableData.value.length) return [];
// Get the first item to extract columns
const firstItem = tableData.value[0];
if (typeof firstItem !== 'object' || firstItem === null) return [];
// Extract columns from object keys
return Object.keys(firstItem).map(key => ({
prop: key,
label: key.charAt(0).toUpperCase() + key.slice(1) // Capitalize first letter
}));
});
// Format page data for display (for non-tabular data)
const formattedPageData = computed(() => {
if (!props.pageData || isTabularData.value) return null;
let displayValue = props.pageData;
// Format based on type
if (typeof displayValue === 'object') {
// For object or array, stringify with formatting
try {
displayValue = JSON.stringify(displayValue, null, 2);
} catch (e) {
displayValue = String(displayValue);
}
}
return displayValue;
});
defineOptions({ name: 'ClAutoProbeFieldDetail' });
</script>
<template>
<div class="cl-autoprobe-field-detail">
<div class="header">
<h3>{{ t('components.autoprobe.field.title') }}: {{ field.label }}</h3>
</div>
<div v-if="fieldData" class="content">
<el-descriptions :column="1" border>
<el-descriptions-item :label="t('components.autoprobe.field.name')">
{{ fieldData.name }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.field.selector')">
{{ displaySelector }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.field.type')">
{{ fieldData.selector_type }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.field.extractionType')">
{{ fieldData.extraction_type }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.field.attributeName')" v-if="fieldData.attribute_name">
{{ fieldData.attribute_name }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.field.defaultValue')" v-if="fieldData.default_value">
{{ fieldData.default_value }}
</el-descriptions-item>
</el-descriptions>
<!-- Tabular Data Display -->
<div v-if="isTabularData" class="page-data-section">
<h4>{{ t('components.autoprobe.field.data') || 'Field Data' }}</h4>
<div class="table">
<el-table
:data="tableData"
border
style="width: 100%"
height="400"
highlight-current-row
>
<el-table-column
v-for="column in tableColumns"
:key="column.prop"
:prop="column.prop"
:label="column.label"
sortable
>
<template #default="scope">
<template v-if="typeof scope.row[column.prop] !== 'object' || scope.row[column.prop] === null">
{{ scope.row[column.prop] }}
</template>
<pre v-else class="json-value">{{ JSON.stringify(scope.row[column.prop], null, 2) }}</pre>
</template>
</el-table-column>
</el-table>
<div class="table-footer">
<span class="total-items">
{{ `Total: ${tableData.length} ${tableData.length === 1 ? 'item' : 'items'}` }}
</span>
</div>
</div>
</div>
<!-- Single Value or Non-tabular Data -->
<div v-else-if="pageData" class="page-data-section">
<h4>{{ t('components.autoprobe.field.pageData') || 'Page Data' }}</h4>
<el-card shadow="never" class="page-data-card">
<pre v-if="typeof formattedPageData === 'string' && (formattedPageData.startsWith('{') || formattedPageData.startsWith('['))" class="json-value">{{ formattedPageData }}</pre>
<span v-else>{{ formattedPageData }}</span>
</el-card>
</div>
</div>
<div v-else class="not-found">
{{ t('components.autoprobe.field.notFound') || 'Field details not found' }}
</div>
</div>
</template>
<style scoped>
.cl-autoprobe-field-detail {
padding: 16px;
border: 1px solid var(--el-border-color);
border-radius: 4px;
.header {
margin-bottom: 16px;
padding-bottom: 8px;
border-bottom: 1px solid var(--el-border-color-light);
h3 {
margin: 0;
font-size: 16px;
font-weight: 500;
}
}
.content {
width: 100%;
.page-data-section {
margin-top: 20px;
h4 {
margin: 0 0 12px 0;
font-size: 14px;
font-weight: 500;
}
.page-data-card {
border: 1px solid var(--el-border-color-light);
}
}
.table {
width: 100%;
.el-table__inner-wrapper {
position: relative;
overflow: unset;
}
.el-table__header-wrapper {
position: sticky;
top: 0;
z-index: 1;
}
.table-footer {
padding: 8px 12px;
display: flex;
justify-content: space-between;
align-items: center;
border-top: 1px solid var(--el-border-color);
.total-items {
font-size: 12px;
color: var(--el-text-color-secondary);
}
}
}
.json-value {
margin: 0;
white-space: pre-wrap;
word-break: break-word;
font-family: monospace;
font-size: 12px;
max-height: 150px;
overflow-y: auto;
}
}
.not-found {
color: var(--el-text-color-secondary);
font-style: italic;
text-align: center;
padding: 20px;
}
}
</style>

181
frontend/crawlab-ui/src/components/core/autoprobe/AutoProbeItemDetail.vue Normal file
View File

@@ -0,0 +1,181 @@
<script setup lang="tsx">
import { computed } from 'vue';
import { ClNavLink, ClIcon, ClAutoProbeSelector } from '@/components';
import { translate } from '@/utils';
import { CellStyle, ColumnStyle } from 'element-plus';
const props = defineProps<{
item: AutoProbeNavItem;
activeId?: string;
}>();
const emit = defineEmits<{
(e: 'row-click', id: string): void;
}>();
const t = translate;
const tableColumns = computed<TableColumns<AutoProbeNavItem>>(() => {
return [
{
key: 'name',
label: t('components.autoprobe.pagePattern.name'),
width: '200',
value: (row: AutoProbeNavItem) => {
let icon: Icon;
switch (row.type) {
case 'field':
icon = ['fa', 'tag'];
break;
case 'list':
icon = ['fa', 'list'];
break;
case 'pagination':
icon = ['fa', 'ellipsis-h'];
break;
default:
icon = ['fa', 'question'];
}
return (
<ClNavLink
onClick={() => {
emit('row-click', row.id);
}}
>
<span style={{ marginRight: '5px' }}>
<ClIcon icon={icon} />
</span>
{row.label}
</ClNavLink>
);
},
},
{
key: 'type',
label: t('components.autoprobe.pagePattern.type'),
width: '100',
value: (row: AutoProbeNavItem) => {
return t(`components.autoprobe.pagePattern.types.${row.type}`);
},
},
{
key: 'fields',
label: t('components.autoprobe.pagePattern.fieldCount'),
width: '80',
value: (row: AutoProbeNavItem) => {
if (row.type === 'list') {
return (
<ClNavLink
label={row.fieldCount}
onClick={() => {
emit('row-click', row.id);
}}
/>
);
}
},
},
{
key: 'selector',
label: t('components.autoprobe.pagePattern.selector'),
width: 'auto',
minWidth: '300',
value: (row: AutoProbeNavItem) => {
switch (row.type) {
case 'list':
const list = row.rule as ListRule;
const selectorType = list.item_selector_type;
const selector = [list.item_selector, list.list_selector]
.filter(item => item)
.join(' > ');
return (
<ClAutoProbeSelector
selectorType={selectorType}
selector={selector}
onClick={() => {
emit('row-click', row.id);
}}
/>
);
case 'field':
const field = row.rule as FieldRule;
return (
<ClAutoProbeSelector
selectorType={field.selector_type}
selector={field.selector}
extractType={field.extraction_type}
attribute={field.attribute_name}
onClick={() => {
emit('row-click', row.id);
}}
/>
);
case 'pagination':
const pagination = row.rule as PaginationRule;
return (
<ClAutoProbeSelector
selectorType={pagination.selector_type}
selector={pagination.selector}
onClick={() => {
emit('row-click', row.id);
}}
/>
);
}
},
},
] as TableColumns<AutoProbeNavItem>;
});
const tableData = computed<TableData<AutoProbeNavItem>>(() => {
const { item } = props;
if (!item.children) return [];
return item?.children.map((item: AutoProbeNavItem) => {
const row = {
id: item.id,
name: item.name,
label: item.name,
type: item.type,
rule: item.rule,
} as AutoProbeNavItem;
if (item.type === 'list') {
row.fieldCount = item.children?.length || 0;
}
return row;
}) as TableData<AutoProbeNavItem>;
});
const rowStyle: ColumnStyle<AutoProbeNavItem> = ({ row }) => {
const { activeId } = props;
if (row.id === activeId) {
return {
backgroundColor: 'var(--el-color-primary-light-9)',
};
}
return {};
};
const cellStyle: CellStyle<AutoProbeNavItem> = ({ row }) => {
const { activeId } = props;
if (row.id === activeId) {
return {
backgroundColor: 'var(--el-color-primary-light-9)',
};
}
return {};
};
defineOptions({ name: 'ClAutoProbeItemDetail' });
</script>
<template>
<div class="cl-autoprobe-page-pattern-detail">
<cl-table
:columns="tableColumns"
:data="tableData"
embedded
hide-footer
:row-style="rowStyle"
:cell-style="cellStyle"
/>
</div>
</template>

359
frontend/crawlab-ui/src/components/core/autoprobe/AutoProbeListDetail.vue
View File

@@ -1,359 +0,0 @@
<script setup lang="ts">
import { computed } from 'vue';
import { useStore } from 'vuex';
import { translate } from '@/utils';
// i18n
const t = translate;
// props
const props = defineProps<{
list: AutoProbeNavItem;
pageData?: any;
}>();
// store
const store = useStore();
const { autoprobe: state } = store.state as RootStoreState;
const form = computed<AutoProbe>(() => state.form);
// Find the actual list data from the form based on the nav item
const listData = computed<ListRule | undefined>(() => {
if (!form.value?.page_pattern) return undefined;
const listName = props.list.id;
// Find the list recursively
const findList = (lists: ListRule[] | undefined): ListRule | undefined => {
if (!lists) return undefined;
for (const list of lists) {
if (list.name === listName) return list;
const nestedResult = findList(list.item_pattern?.lists);
if (nestedResult) return nestedResult;
}
return undefined;
};
return findList(form.value.page_pattern.lists);
});
// Get raw array data for direct table display (when page data is an array)
const tableData = computed(() => {
if (!props.pageData) return [];
// If page data is an array, it's likely the list data itself
if (Array.isArray(props.pageData)) {
return props.pageData;
}
// If it's an object with a property matching the list id, try that
else if (typeof props.pageData === 'object' && props.pageData !== null &&
Array.isArray(props.pageData[props.list.id])) {
return props.pageData[props.list.id];
}
return [];
});
// Get table columns dynamically based on data
const tableColumns = computed(() => {
if (!tableData.value.length) return [];
// Get the first item to determine columns
const firstItem = tableData.value[0];
if (typeof firstItem !== 'object' || firstItem === null) {
// Simple value array
return [{ prop: 'value', label: 'Value' }];
}
// Extract columns from object properties
return Object.keys(firstItem).map(key => ({
prop: key,
label: key.charAt(0).toUpperCase() + key.slice(1) // Capitalize first letter
}));
});
// Determine if we should show the data table
const showDataTable = computed(() => {
return tableData.value.length > 0 && tableColumns.value.length > 0;
});
// Format page data for display (key-value format for non-array data)
const formattedPageData = computed(() => {
if (!props.pageData || showDataTable.value) return [];
// Convert the page data to a format suitable for display
const result = [];
// If it's an object, display key-value pairs
if (typeof props.pageData === 'object' && !Array.isArray(props.pageData)) {
for (const key in props.pageData) {
if (Object.prototype.hasOwnProperty.call(props.pageData, key)) {
const value = props.pageData[key];
let displayValue = value;
// Format based on value type
if (typeof value === 'object') {
try {
displayValue = JSON.stringify(value, null, 2);
} catch (e) {
displayValue = String(value);
}
}
result.push({
key,
value: displayValue
});
}
}
}
// If it's an array but not suitable for table display, process each item
else if (Array.isArray(props.pageData)) {
props.pageData.forEach((item, index) => {
let displayValue = item;
// Format based on type
if (typeof item === 'object') {
try {
displayValue = JSON.stringify(item, null, 2);
} catch (e) {
displayValue = String(item);
}
}
result.push({
key: `Item ${index + 1}`,
value: displayValue
});
});
}
// For primitive types
else {
result.push({
key: 'value',
value: props.pageData
});
}
return result;
});
// Compute fields and nested lists for display
const fields = computed(() => listData.value?.item_pattern?.fields || []);
const nestedLists = computed(() => listData.value?.item_pattern?.lists || []);
// Display value for list selector - if empty, it points to itself (self)
const displayListSelector = computed(() => {
if (!listData.value?.list_selector) return t('components.autoprobe.list.self') || 'self';
return listData.value.list_selector;
});
// Display value for item selector - if empty, it points to itself (self)
const displayItemSelector = computed(() => {
if (!listData.value?.item_selector) return t('components.autoprobe.list.self') || 'self';
return listData.value.item_selector;
});
// Process fields for display, marking empty selectors as "self"
const processedFields = computed(() => {
return fields.value.map(field => ({
...field,
displaySelector: field.selector || (t('components.autoprobe.field.self') || 'self')
}));
});
// Process nested lists for display, marking empty selectors as "self"
const processedNestedLists = computed(() => {
return nestedLists.value.map(nestedList => ({
...nestedList,
displayListSelector: nestedList.list_selector || (t('components.autoprobe.list.self') || 'self')
}));
});
defineOptions({ name: 'ClAutoProbeListDetail' });
</script>
<template>
<div class="cl-autoprobe-list-detail">
<div class="header">
<h3>{{ t('components.autoprobe.list.title') }}: {{ list.label }}</h3>
</div>
<div v-if="listData" class="content">
<el-descriptions :column="1" border>
<el-descriptions-item :label="t('components.autoprobe.list.name')">
{{ listData.name }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.list.listSelector')">
{{ displayListSelector }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.list.listSelectorType')">
{{ listData.list_selector_type }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.list.itemSelector')">
{{ displayItemSelector }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.list.itemSelectorType')">
{{ listData.item_selector_type }}
</el-descriptions-item>
</el-descriptions>
<div v-if="fields.length" class="fields-section">
<h4>{{ t('components.autoprobe.list.fields') }}</h4>
<el-table :data="processedFields" border style="width: 100%">
<el-table-column prop="name" :label="t('components.autoprobe.field.name')" width="200" />
<el-table-column :label="t('components.autoprobe.field.selector')">
<template #default="scope">
{{ scope.row.displaySelector }}
</template>
</el-table-column>
</el-table>
</div>
<div v-if="nestedLists.length" class="nested-lists-section">
<h4>{{ t('components.autoprobe.list.nestedLists') }}</h4>
<el-table :data="processedNestedLists" border style="width: 100%">
<el-table-column prop="name" :label="t('components.autoprobe.list.name')" width="200" />
<el-table-column :label="t('components.autoprobe.list.listSelector')">
<template #default="scope">
{{ scope.row.displayListSelector }}
</template>
</el-table-column>
</el-table>
</div>
<!-- Dynamic Data Table for List Items -->
<div v-if="showDataTable" class="page-data-section">
<h4>{{ t('components.autoprobe.list.items') || 'List Items' }}</h4>
<div class="table">
<el-table
:data="tableData"
border
style="width: 100%"
height="400"
highlight-current-row
>
<el-table-column
v-for="column in tableColumns"
:key="column.prop"
:prop="column.prop"
:label="column.label"
sortable
>
<template #default="scope">
<template v-if="typeof scope.row[column.prop] !== 'object' || scope.row[column.prop] === null">
{{ scope.row[column.prop] }}
</template>
<pre v-else class="json-value">{{ JSON.stringify(scope.row[column.prop], null, 2) }}</pre>
</template>
</el-table-column>
</el-table>
<div class="table-footer">
<span class="total-items">
{{ `Total: ${tableData.length} ${tableData.length === 1 ? 'item' : 'items'}` }}
</span>
</div>
</div>
</div>
<!-- Page Data Section (for non-tabular data) -->
<div v-else-if="formattedPageData.length" class="page-data-section">
<h4>{{ t('components.autoprobe.list.pageData') || 'Page Data' }}</h4>
<el-table :data="formattedPageData" border style="width: 100%">
<el-table-column prop="key" :label="t('components.autoprobe.pageData.key') || 'Key'" width="200" />
<el-table-column :label="t('components.autoprobe.pageData.value') || 'Value'">
<template #default="scope">
<pre v-if="typeof scope.row.value === 'string' && (scope.row.value.startsWith('{') || scope.row.value.startsWith('['))" class="json-value">{{ scope.row.value }}</pre>
<span v-else>{{ scope.row.value }}</span>
</template>
</el-table-column>
</el-table>
</div>
</div>
<div v-else class="not-found">
{{ t('components.autoprobe.list.notFound') || 'List details not found' }}
</div>
</div>
</template>
<style scoped>
.cl-autoprobe-list-detail {
padding: 16px;
border: 1px solid var(--el-border-color);
border-radius: 4px;
.header {
margin-bottom: 16px;
padding-bottom: 8px;
border-bottom: 1px solid var(--el-border-color-light);
h3 {
margin: 0;
font-size: 16px;
font-weight: 500;
}
}
.content {
width: 100%;
.fields-section,
.nested-lists-section,
.page-data-section {
margin-top: 20px;
h4 {
margin: 0 0 12px 0;
font-size: 14px;
font-weight: 500;
}
}
.table {
width: 100%;
.el-table__inner-wrapper {
position: relative;
overflow: unset;
}
.el-table__header-wrapper {
position: sticky;
top: 0;
z-index: 1;
}
.table-footer {
padding: 8px 12px;
display: flex;
justify-content: space-between;
align-items: center;
border-top: 1px solid var(--el-border-color);
.total-items {
font-size: 12px;
color: var(--el-text-color-secondary);
}
}
}
.json-value {
margin: 0;
white-space: pre-wrap;
word-break: break-word;
font-family: monospace;
font-size: 12px;
max-height: 150px;
overflow-y: auto;
}
}
.not-found {
color: var(--el-text-color-secondary);
font-style: italic;
text-align: center;
padding: 20px;
}
}
</style>

236
frontend/crawlab-ui/src/components/core/autoprobe/AutoProbePagePatternDetail.vue
View File

@@ -1,236 +0,0 @@
<script setup lang="tsx">
import { computed } from 'vue';
import { useStore } from 'vuex';
import {
ClNavLink,
ClIcon,
ClAutoProbeSelector,
} from '@/components';
import { translate } from '@/utils';
const props = defineProps<{
pagePattern: AutoProbeNavItem;
}>();
const t = translate;
// store
const store = useStore();
const { autoprobe: state } = store.state as RootStoreState;
const form = computed<AutoProbe>(() => state.form);
const tableColumns = computed<TableColumns<AutoProbeNavItem>>(() => {
return [
{
key: 'name',
label: t('components.autoprobe.pagePattern.name'),
width: '200',
value: (row: AutoProbeNavItem) => {
let icon: Icon;
switch (row.type) {
case 'field':
icon = ['fa', 'tag'];
break;
case 'list':
icon = ['fa', 'list'];
break;
case 'pagination':
icon = ['fa', 'ellipsis-h'];
break;
default:
icon = ['fa', 'question'];
}
return (
<ClNavLink onClick={() => {}}>
<span style={{ marginRight: '5px' }}>
<ClIcon icon={icon} />
</span>
{row.label}
</ClNavLink>
);
},
},
{
key: 'type',
label: t('components.autoprobe.pagePattern.type'),
width: '100',
value: (row: AutoProbeNavItem) => {
return t(`components.autoprobe.pagePattern.types.${row.type}`);
},
},
{
key: 'fields',
label: t('components.autoprobe.pagePattern.fieldCount'),
width: '80',
value: (row: AutoProbeNavItem) => {
if (row.type === 'list') {
return <ClNavLink label={row.fieldCount} onClick={() => {}} />;
}
},
},
{
key: 'selector',
label: t('components.autoprobe.pagePattern.selector'),
width: 'auto',
minWidth: '300',
value: (row: AutoProbeNavItem) => {
switch (row.type) {
case 'field':
return <ClAutoProbeSelector type="field" rule={row.field} />;
case 'pagination':
return <ClAutoProbeSelector type="pagination" rule={row.pagination} />;
}
},
},
] as TableColumns<AutoProbeNavItem>;
});
const tableData = computed<TableData<AutoProbeNavItem>>(() => {
const { pagePattern } = props;
if (!pagePattern?.children?.length) {
return [];
}
return pagePattern.children.map((item: AutoProbeNavItem) => {
switch (item.type) {
case 'list':
return {
name: item.name,
label: item.name,
type: item.type,
fieldCount: item.children?.length || 0,
};
case 'field':
return {
name: item.name,
label: item.name,
type: item.type,
field: item.field,
};
case 'pagination':
return {
name: item.name,
label: item.name,
type: item.type,
pagination: item.pagination,
};
default:
return {
name: item.name,
label: item.name,
type: item.type,
};
}
}) as TableData<AutoProbeNavItem>;
});
defineOptions({ name: 'ClAutoProbePagePatternDetail' });
</script>
<template>
<div class="cl-autoprobe-page-pattern-detail">
<cl-table :columns="tableColumns" :data="tableData" embedded hide-footer />
</div>
</template>
<style scoped>
.cl-autoprobe-page-pattern-detail {
.header {
margin-bottom: 16px;
padding-bottom: 8px;
border-bottom: 1px solid var(--el-border-color-light);
h3 {
margin: 0;
font-size: 16px;
font-weight: 500;
}
}
.content {
width: 100%;
.stats-section {
margin-top: 20px;
h4 {
margin: 0 0 12px 0;
font-size: 14px;
font-weight: 500;
}
.stat-card {
background-color: var(--el-fill-color-light);
border-radius: 4px;
padding: 16px;
text-align: center;
height: 100%;
.stat-value {
font-size: 24px;
font-weight: 500;
margin-bottom: 8px;
}
.stat-label {
color: var(--el-text-color-secondary);
font-size: 14px;
}
}
}
.page-data-section {
margin-top: 20px;
h4 {
margin: 0 0 12px 0;
font-size: 14px;
font-weight: 500;
}
}
.table {
width: 100%;
.el-table__inner-wrapper {
position: relative;
overflow: unset;
}
.el-table__header-wrapper {
position: sticky;
top: 0;
z-index: 1;
}
.table-footer {
padding: 8px 12px;
display: flex;
justify-content: space-between;
align-items: center;
border-top: 1px solid var(--el-border-color);
.total-items {
font-size: 12px;
color: var(--el-text-color-secondary);
}
}
}
.json-value {
margin: 0;
white-space: pre-wrap;
word-break: break-word;
font-family: monospace;
font-size: 12px;
max-height: 150px;
overflow-y: auto;
}
}
.not-found {
color: var(--el-text-color-secondary);
font-style: italic;
text-align: center;
padding: 20px;
}
}
</style>

315
frontend/crawlab-ui/src/components/core/autoprobe/AutoProbePagePatternsSidebar.vue Normal file
View File

@@ -0,0 +1,315 @@
<script setup lang="tsx">
import { computed, ref, watch } from 'vue';
import { ElTree, ElInput, ElScrollbar } from 'element-plus';
import type { FilterNodeMethodFunction } from 'element-plus/es/components/tree/src/tree.type';
import { debounce } from 'lodash';
import { translate } from '@/utils';
// i18n
const t = translate;
// props and emits
const props = defineProps<{
activeNavItemId?: string;
treeItems?: AutoProbeNavItem[];
defaultExpandedKeys?: string[];
}>();
const emit = defineEmits<{
(e: 'node-select', item: AutoProbeNavItem): void;
}>();
const treeRef = ref<InstanceType<typeof ElTree>>();
const searchKeyword = ref('');
const showSearch = ref(false);
const defaultExpandedKeys = ref<string[]>(props.defaultExpandedKeys || []);
// Context menu functionality
const activeContextMenuNavItem = ref<AutoProbeNavItem>();
const contextMenuVisibleMap = ref<Record<string, boolean>>({});
const isContextMenuVisible = (id: string) => {
if (!contextMenuItems.value?.length) return false;
if (!activeContextMenuNavItem.value) return false;
if (activeContextMenuNavItem.value?.id !== id) return false;
return contextMenuVisibleMap.value[id] || false;
};
const onActionsClick = (item: AutoProbeNavItem) => {
activeContextMenuNavItem.value = item;
contextMenuVisibleMap.value[item.id] = true;
};
const onContextMenuHide = (id: string) => {
activeContextMenuNavItem.value = undefined;
contextMenuVisibleMap.value[id] = false;
};
const contextMenuItems = computed(() => {
if (!activeContextMenuNavItem.value) return [];
const { id, type } = activeContextMenuNavItem.value;
if (!contextMenuVisibleMap.value[id]) return [];
switch (type) {
case 'field':
return [
{
title: t('common.actions.view'),
icon: ['fa', 'eye'],
action: () =>
selectNode(activeContextMenuNavItem.value as AutoProbeNavItem),
},
];
case 'list':
return [
{
title: t('common.actions.view'),
icon: ['fa', 'eye'],
action: () =>
selectNode(activeContextMenuNavItem.value as AutoProbeNavItem),
},
];
default:
return [];
}
});
// Search functionality
const onSearchFilter: FilterNodeMethodFunction = (value, data) => {
if (!value) return true;
return data.label.toLowerCase().includes(value.toLowerCase());
};
const debouncedFilter = debounce(() => {
treeRef.value?.filter(searchKeyword.value);
}, 300);
watch(searchKeyword, debouncedFilter);
// Node selection and expansion tracking
const onNodeClick = async (data: AutoProbeNavItem) => {
await selectNode(data);
};
const selectNode = async (data: AutoProbeNavItem) => {
const { id } = data;
emit('node-select', data);
// Highlight current node
setTimeout(() => {
treeRef.value?.setCurrentKey(id);
}, 0);
};
const onNodeExpand = (data: AutoProbeNavItem) => {
defaultExpandedKeys.value.push(data.id);
};
const onNodeCollapse = (data: AutoProbeNavItem) => {
const idx = defaultExpandedKeys.value.findIndex(id => id === data.id);
defaultExpandedKeys.value.splice(idx, 1);
};
const getNode = (id: string) => {
return treeRef.value?.getNode(id)?.data as AutoProbeNavItem | undefined;
};
const onContextMenuClick = (event: MouseEvent, data: AutoProbeNavItem) => {
event.stopPropagation();
activeContextMenuNavItem.value = data;
contextMenuVisibleMap.value[data.id] = true;
};
const onSearchClick = () => {
showSearch.value = !showSearch.value;
};
// Update current selection when prop changes
watch(
() => props.activeNavItemId,
newId => {
if (newId) {
setTimeout(() => {
treeRef.value?.setCurrentKey(newId);
}, 0);
}
}
);
// Sidebar resizing
const widthKey = ref('autoprobe.sidebar.width');
const sidebarRef = ref<HTMLElement | null>(null);
defineExpose({
getNode,
});
defineOptions({ name: 'ClAutoProbePagePatternsSidebar' });
</script>
<template>
<div ref="sidebarRef" class="sidebar">
<cl-resize-handle :target-ref="sidebarRef" :size-key="widthKey" />
<div class="sidebar-actions">
<cl-icon
:class="showSearch ? 'selected' : ''"
:icon="['fa', 'search']"
@click="onSearchClick"
/>
</div>
<div v-if="showSearch" class="sidebar-search">
<el-input
v-model="searchKeyword"
:placeholder="t('common.actions.search')"
clearable
@clear="
() => {
searchKeyword = '';
showSearch = false;
}
"
/>
</div>
<el-scrollbar>
<el-tree
ref="treeRef"
node-key="id"
:data="treeItems"
:filter-node-method="onSearchFilter"
:expand-on-click-node="false"
:default-expanded-keys="defaultExpandedKeys"
highlight-current
@node-click="onNodeClick"
@node-contextmenu="onContextMenuClick"
@node-expand="onNodeExpand"
@node-collapse="onNodeCollapse"
>
<template #default="{ data }">
<cl-context-menu
:visible="isContextMenuVisible(data.id)"
:style="{ flex: 1, paddingRight: '5px' }"
>
<template #reference>
<div class="node-wrapper" :title="data.label">
<span class="icon-wrapper">
<cl-icon
v-if="data.loading"
:icon="['fa', 'spinner']"
spinning
/>
<cl-icon v-else :icon="data.icon || ['fa', 'folder']" />
</span>
<span class="label">
{{ data.label }}
</span>
</div>
<div class="actions">
<cl-icon
class="more"
:icon="['fa', 'ellipsis']"
@click.stop="onActionsClick(data)"
/>
</div>
</template>
<cl-context-menu-list
v-if="isContextMenuVisible(data.id)"
:items="contextMenuItems"
@hide="onContextMenuHide(data.id)"
/>
</cl-context-menu>
</template>
</el-tree>
</el-scrollbar>
</div>
</template>
<style scoped>
.sidebar {
flex: 0 0 240px;
height: 100%;
overflow: hidden;
border-right: 1px solid var(--el-border-color);
display: flex;
flex-direction: column;
position: relative;
.sidebar-actions {
height: 41px;
flex: 0 0 41px;
padding: 5px;
display: flex;
align-items: center;
gap: 5px;
color: var(--cl-primary-color);
border-bottom: 1px solid var(--el-border-color);
& > * {
display: flex;
align-items: center;
}
&:deep(.icon) {
cursor: pointer;
padding: 6px;
font-size: 14px;
width: 14px;
height: 14px;
border-radius: 50%;
}
&:deep(.icon.selected),
&:deep(.icon:hover) {
background-color: var(--cl-primary-plain-color);
}
}
.sidebar-search {
height: 38px;
flex: 0 0 38px;
border-bottom: 1px solid var(--el-border-color);
&:deep(.el-input .el-input__wrapper) {
box-shadow: none;
border: none;
}
}
.el-tree {
min-width: fit-content;
&:deep(.el-tree-node__content:hover .actions .icon) {
display: flex !important;
}
&:deep(.el-tree-node__content) {
width: 100%;
position: relative;
.node-wrapper {
display: flex;
align-items: center;
position: relative;
width: 100%;
.icon-wrapper {
width: 20px;
display: flex;
}
.label {
flex: 0 0 auto;
}
}
.actions {
display: flex;
gap: 5px;
position: absolute;
top: 0;
right: 5px;
height: 100%;
align-items: center;
&:deep(.icon.more) {
display: none;
}
}
}
}
}
</style>

138
frontend/crawlab-ui/src/components/core/autoprobe/AutoProbePaginationDetail.vue
View File

@@ -1,138 +0,0 @@
<script setup lang="ts">
import { computed } from 'vue';
import { useStore } from 'vuex';
import { translate } from '@/utils';
// i18n
const t = translate;
// props
const props = defineProps<{
pagination: AutoProbeNavItem;
pageData?: any;
}>();
// store
const store = useStore();
const { autoprobe: state } = store.state as RootStoreState;
const form = computed<AutoProbe>(() => state.form);
// Pagination data
const paginationData = computed(() => form.value?.page_pattern?.pagination);
// Format page data for display
const formattedPageData = computed(() => {
if (!props.pageData) return null;
let displayValue = props.pageData;
// Format based on type
if (typeof displayValue === 'object') {
// For object or array, stringify with formatting
try {
displayValue = JSON.stringify(displayValue, null, 2);
} catch (e) {
displayValue = String(displayValue);
}
}
return displayValue;
});
defineOptions({ name: 'ClAutoProbePaginationDetail' });
</script>
<template>
<div class="cl-autoprobe-pagination-detail">
<div class="header">
<h3>{{ t('components.autoprobe.pagination.title') }}</h3>
</div>
<div v-if="paginationData" class="content">
<el-descriptions :column="1" border>
<el-descriptions-item :label="t('components.autoprobe.pagination.type')">
{{ paginationData.type }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.pagination.selectorType')" v-if="paginationData.selector_type">
{{ paginationData.selector_type }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.pagination.selector')" v-if="paginationData.selector">
{{ paginationData.selector }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.pagination.maxPages')" v-if="paginationData.max_pages">
{{ paginationData.max_pages }}
</el-descriptions-item>
<el-descriptions-item :label="t('components.autoprobe.pagination.startPage')" v-if="paginationData.start_page">
{{ paginationData.start_page }}
</el-descriptions-item>
</el-descriptions>
<!-- Page Data Section -->
<div v-if="pageData" class="page-data-section">
<h4>{{ t('components.autoprobe.pagination.pageData') || 'Page Data' }}</h4>
<el-card shadow="never" class="page-data-card">
<pre v-if="typeof formattedPageData === 'string' && formattedPageData.startsWith('{')" class="json-value">{{ formattedPageData }}</pre>
<pre v-else-if="typeof formattedPageData === 'string' && formattedPageData.startsWith('[')" class="json-value">{{ formattedPageData }}</pre>
<span v-else>{{ formattedPageData }}</span>
</el-card>
</div>
</div>
<div v-else class="not-found">
{{ t('components.autoprobe.pagination.notFound') || 'Pagination details not found' }}
</div>
</div>
</template>
<style scoped>
.cl-autoprobe-pagination-detail {
padding: 16px;
border: 1px solid var(--el-border-color);
border-radius: 4px;
.header {
margin-bottom: 16px;
padding-bottom: 8px;
border-bottom: 1px solid var(--el-border-color-light);
h3 {
margin: 0;
font-size: 16px;
font-weight: 500;
}
}
.content {
width: 100%;
.page-data-section {
margin-top: 20px;
h4 {
margin: 0 0 12px 0;
font-size: 14px;
font-weight: 500;
}
.page-data-card {
border: 1px solid var(--el-border-color-light);
}
}
.json-value {
margin: 0;
white-space: pre-wrap;
word-break: break-word;
font-family: monospace;
font-size: 12px;
}
}
.not-found {
color: var(--el-text-color-secondary);
font-style: italic;
text-align: center;
padding: 20px;
}
}
</style>

208
frontend/crawlab-ui/src/components/core/autoprobe/AutoProbeResultsContainer.vue Normal file
View File

@@ -0,0 +1,208 @@
<script setup lang="tsx">
import { computed, ref } from 'vue';
import { ClTag } from '@/components';
import { translate, getIconByItemType } from '@/utils';
import { TAB_NAME_RESULTS, TAB_NAME_PREVIEW } from '@/constants';
const t = translate;
// Props
const props = defineProps<{
data?: PageData | PageData[];
fields?: AutoProbeNavItem[];
}>();
// Refs
const resultsContainerRef = ref<HTMLElement | null>(null);
// States
const activeTabName = ref<string | undefined>(TAB_NAME_RESULTS);
// Computed
const resultsVisible = computed(() => !!activeTabName.value);
const resultsTabItems = computed<NavItem[]>(() => [
{
id: TAB_NAME_RESULTS,
title: t('common.tabs.results'),
},
{
id: TAB_NAME_PREVIEW,
title: t('common.tabs.preview'),
},
]);
const tableColumns = computed<TableColumns<PageData>>(() => {
const { fields } = props;
if (!fields) return [];
return fields.map(field => {
return {
key: field.name,
label: field.name,
value: (row: PageData) => {
switch (field.type) {
case 'list':
return (
<ClTag
icon={getIconByItemType('list')}
label={field.children?.length || 0}
/>
);
default:
return row[field.name!];
}
},
};
}) as TableColumns<PageData>;
});
const tableData = computed<TableData<PageData | PageData[]>>(() => {
const { data } = props;
if (!data) return [];
if (Array.isArray(data)) {
return data;
}
return [data];
});
// Methods
const onTabSelect = (id: string) => {
if (activeTabName.value === id) {
hideResults();
} else {
activeTabName.value = id;
}
};
const hideResults = () => {
activeTabName.value = undefined;
};
// Resize handler
const heightKey = 'autoprobe.results.containerHeight';
const onSizeChange = (size: number) => {
// Emit event to parent to adjust layout
emit('size-change', size);
};
// Emits
const emit = defineEmits<{
(e: 'size-change', size: number): void;
}>();
defineOptions({ name: 'ClAutoProbeResultsContainer' });
</script>
<template>
<div
ref="resultsContainerRef"
class="autoprobe-results-container"
:class="[resultsVisible ? 'results-visible' : '']"
>
<cl-resize-handle
v-if="resultsVisible"
:target-ref="resultsContainerRef"
:size-key="heightKey"
direction="horizontal"
position="start"
@size-change="onSizeChange"
/>
<cl-nav-tabs
:active-key="activeTabName"
:items="resultsTabItems"
@select="onTabSelect"
>
<template #extra>
<div class="results-actions">
<cl-icon
v-if="resultsVisible"
color="var(--cl-info-color)"
:icon="['fa', 'minus']"
@click="hideResults"
/>
</div>
</template>
</cl-nav-tabs>
<div class="results" v-if="activeTabName === TAB_NAME_RESULTS">
<cl-table
:columns="tableColumns"
:data="tableData"
embedded
hide-footer
/>
</div>
<div class="output" v-else-if="activeTabName === TAB_NAME_PREVIEW"></div>
</div>
</template>
<style scoped>
.autoprobe-results-container {
position: relative;
border-top: 1px solid var(--el-border-color);
overflow: hidden;
&.results-visible {
overflow: auto;
flex: 0 0 50%;
height: 50%;
}
&:not(.results-visible) {
flex: 0 0 41px !important;
height: 41px !important;
}
.results-actions {
display: flex;
align-items: center;
padding: 0 10px;
&:deep(.icon) {
cursor: pointer;
padding: 6px;
font-size: 14px;
width: 14px;
height: 14px;
border-radius: 50%;
}
&:deep(.icon:hover) {
background-color: var(--cl-info-plain-color);
}
}
.results {
height: calc(100% - 41px);
&:deep(.table) {
width: 100%;
height: 100%;
}
&:deep(.table .el-table__inner-wrapper) {
position: relative;
overflow: unset;
}
&:deep(.table .el-table__header-wrapper) {
position: sticky;
top: 0;
}
}
.output {
padding: 10px;
height: calc(100% - 41px);
overflow: auto;
white-space: pre-wrap;
pre {
margin: 0;
font-size: 14px;
line-height: 1.5;
color: var(--cl-text-color);
white-space: pre-wrap;
}
}
}
</style>

115
frontend/crawlab-ui/src/components/core/autoprobe/AutoProbeSelector.vue
View File

@@ -1,53 +1,32 @@
<script setup lang="ts">
import { computed } from 'vue';
import { translate } from '@/utils';
import {
translate,
getIconBySelectorType,
getIconByExtractType,
} from '@/utils';
const props = defineProps<{
type: 'field' | 'pagination';
rule: FieldRule | Pagination;
selectorType: SelectorType;
selector: string;
extractType?: ExtractType;
attribute?: string;
}>();
const emit = defineEmits<{
(e: 'click'): void;
}>();
const t = translate;
const selectorIcon = computed<Icon>(() => {
const { rule } = props;
switch (rule.selector_type) {
case 'css':
return ['fab', 'css'];
case 'xpath':
return ['fa', 'code'];
case 'regex':
return ['fa', 'search'];
}
const { selectorType } = props;
return getIconBySelectorType(selectorType);
});
const selectorType = computed(() => {});
const extractionIcon = computed<Icon>(() => {
const { rule, type } = props;
if (type === 'field') {
switch ((rule as FieldRule).extraction_type) {
case 'attribute':
return ['fa', 'tag'];
case 'text':
return ['fa', 'font'];
case 'html':
return ['fa', 'code'];
default:
return ['fa', 'question'];
}
} else {
return ['fa', 'question'];
}
});
const extractionLabel = computed(() => {
const { rule, type } = props;
if (type === 'field') {
return (rule as FieldRule).extraction_type;
} else {
return '';
}
const extractIcon = computed<Icon>(() => {
const { extractType } = props;
return getIconByExtractType(extractType);
});
defineOptions({ name: 'ClAutoProbeSelector' });
@@ -57,18 +36,62 @@ defineOptions({ name: 'ClAutoProbeSelector' });
<div class="selector">
<cl-tag
:icon="selectorIcon"
:label="rule.selector"
:label="selector"
:tooltip="
t(
`components.autoprobe.pagePattern.selectorTypes.${rule.selector_type}`
)
t(`components.autoprobe.pagePattern.selectorTypes.${selectorType}`)
"
/>
<template v-if="type === 'field'">
clickable
>
<template #tooltip>
<div>
<label>
{{ t('components.autoprobe.pagePattern.selectorType') }}:
</label>
<span>
{{
t(
`components.autoprobe.pagePattern.selectorTypes.${selectorType}`
)
}}
</span>
</div>
<div>
<label>{{ t('components.autoprobe.pagePattern.selector') }}: </label>
<span>{{ selector }}</span>
</div>
</template>
</cl-tag>
<template v-if="extractType">
<span class="divider">
<cl-icon :icon="['fa', 'angle-right']" />
</span>
<cl-tag :icon="extractionIcon" :label="extractionLabel" />
<cl-tag
:icon="extractIcon"
:label="attribute"
clickable
@click="emit('click')"
>
<template #tooltip>
<div>
<label>
{{ t('components.autoprobe.pagePattern.extractionType') }}:
</label>
<span>
{{
t(
`components.autoprobe.pagePattern.extractionTypes.${extractType}`
)
}}
</span>
</div>
<div v-if="extractType === 'attribute'">
<label>
{{ t('components.autoprobe.pagePattern.attribute') }}:
</label>
<span>{{ attribute }}</span>
</div>
</template>
</cl-tag>
</template>
</div>
</template>

19
frontend/crawlab-ui/src/components/core/database/DatabaseSidebar.vue
View File

@@ -148,8 +148,14 @@ const debouncedFilter = debounce(() => {
watch(searchKeyword, debouncedFilter);
const loading = ref(false);
const getMetadata = async () => {
await store.dispatch(`${ns}/getMetadata`, { id: activeId.value });
loading.value = true;
try {
await store.dispatch(`${ns}/getMetadata`, { id: activeId.value });
} finally {
loading.value = false;
}
};
const createTable = async () => {
@@ -703,7 +709,9 @@ const sidebarRef = ref<HTMLElement | null>(null);
"
/>
</div>
<el-scrollbar>
<el-skeleton v-if="loading" :rows="10" animated :throttle="100" />
<el-scrollbar v-else>
<el-tree
ref="treeRef"
node-key="id"
@@ -931,5 +939,12 @@ const sidebarRef = ref<HTMLElement | null>(null);
}
}
}
.el-skeleton {
margin: 10px;
width: calc(100% - 20px);
height: calc(100% - 20px);
overflow: hidden;
}
}
</style>

8
frontend/crawlab-ui/src/components/core/spider/SpiderStat.vue
View File

@@ -104,3 +104,11 @@ defineOptions({ name: 'ClSpiderStat' });
</cl-tag>
</div>
</template>
<style scoped>
.spider-stat {
display: flex;
align-items: center;
gap: 5px;
}
</style>

14
frontend/crawlab-ui/src/components/index.ts
View File

@@ -19,12 +19,11 @@ import * as theme from './ui/lexical/utils/theme';
import * as VariableNode from './ui/lexical/nodes/VariableNode';
import AssistantConsole from './core/ai/AssistantConsole.vue';
import AtomMaterialIcon from './ui/icon/AtomMaterialIcon.vue';
import AutoProbeFieldDetail from './core/autoprobe/AutoProbeFieldDetail.vue';
import AutoProbeForm from './core/autoprobe/AutoProbeForm.vue';
import AutoProbeListDetail from './core/autoprobe/AutoProbeListDetail.vue';
import AutoProbePagePatternDetail from './core/autoprobe/AutoProbePagePatternDetail.vue';
import AutoProbePaginationDetail from './core/autoprobe/AutoProbePaginationDetail.vue';
import AutoProbeItemDetail from './core/autoprobe/AutoProbeItemDetail.vue';
import AutoProbePagePatternsSidebar from './core/autoprobe/AutoProbePagePatternsSidebar.vue';
import AutoProbePatternStats from './core/autoprobe/AutoProbePatternStats.vue';
import AutoProbeResultsContainer from './core/autoprobe/AutoProbeResultsContainer.vue';
import AutoProbeSelector from './core/autoprobe/AutoProbeSelector.vue';
import AutoProbeTaskStatus from './core/autoprobe/AutoProbeTaskStatus.vue';
import BlockOptionsDropdownList from './ui/lexical/components/BlockOptionsDropdownList.vue';
@@ -275,12 +274,11 @@ export {
VariableNode as VariableNode,
AssistantConsole as ClAssistantConsole,
AtomMaterialIcon as ClAtomMaterialIcon,
AutoProbeFieldDetail as ClAutoProbeFieldDetail,
AutoProbeForm as ClAutoProbeForm,
AutoProbeListDetail as ClAutoProbeListDetail,
AutoProbePagePatternDetail as ClAutoProbePagePatternDetail,
AutoProbePaginationDetail as ClAutoProbePaginationDetail,
AutoProbeItemDetail as ClAutoProbeItemDetail,
AutoProbePagePatternsSidebar as ClAutoProbePagePatternsSidebar,
AutoProbePatternStats as ClAutoProbePatternStats,
AutoProbeResultsContainer as ClAutoProbeResultsContainer,
AutoProbeSelector as ClAutoProbeSelector,
AutoProbeTaskStatus as ClAutoProbeTaskStatus,
BlockOptionsDropdownList as ClBlockOptionsDropdownList,

1
frontend/crawlab-ui/src/constants/tab.ts
View File

@@ -26,3 +26,4 @@ export const TAB_NAME_ROLES = 'roles';
export const TAB_NAME_PERMISSIONS = 'permissions';
export const TAB_NAME_PAGES = 'pages';
export const TAB_NAME_PATTERNS = 'patterns';
export const TAB_NAME_PREVIEW = 'webpage';

1
frontend/crawlab-ui/src/i18n/lang/en/common.ts
View File

@@ -145,6 +145,7 @@ const common: LCommon = {
results: 'Results',
output: 'Output',
patterns: 'Patterns',
preview: 'Preview',
},
status: {
unassigned: 'Unassigned',

7
frontend/crawlab-ui/src/i18n/lang/en/components/autoprobe.ts
View File

@@ -94,6 +94,13 @@ const autoprobe: LComponentsAutoProbe = {
xpath: 'XPath',
regex: 'Regex',
},
extractionType: 'Extraction Type',
extractionTypes: {
text: 'Text',
attribute: 'Attribute',
html: 'HTML',
},
attribute: 'Attribute',
},
};

1
frontend/crawlab-ui/src/i18n/lang/zh/common.ts
View File

@@ -145,6 +145,7 @@ const common: LCommon = {
results: '结果',
output: '输出',
patterns: '模式',
preview: '预览',
},
status: {
unassigned: '未指定',

7
frontend/crawlab-ui/src/i18n/lang/zh/components/autoprobe.ts
View File

@@ -94,6 +94,13 @@ const autoprobe: LComponentsAutoProbe = {
xpath: 'XPath',
regex: '正则表达式',
},
extractionType: '提取类型',
extractionTypes: {
text: '文本',
html: 'HTML',
attribute: '属性',
},
attribute: '属性',
},
};

1
frontend/crawlab-ui/src/interfaces/i18n/common.d.ts vendored
View File

@@ -158,6 +158,7 @@ export declare global {
results: string;
output: string;
patterns: string;
preview: string;
};
status: {
unassigned: string;

7
frontend/crawlab-ui/src/interfaces/i18n/components/autoprobe.d.ts vendored
View File

@@ -94,5 +94,12 @@ interface LComponentsAutoProbe {
xpath: string;
regex: string;
};
extractionType: string;
extractionTypes: {
text: string;
attribute: string;
html: string;
};
attribute: string;
};
}

30
frontend/crawlab-ui/src/interfaces/models/autoprobe.d.ts vendored
View File

@@ -7,7 +7,7 @@ export declare global {
last_task?: AutoProbeTask;
default_task_id?: string;
page_pattern?: PagePattern;
page_data?: PageData;
page_data?: PageDataRow;
}
type AutoProbeTaskStatus =
@@ -46,19 +46,16 @@ export declare global {
item_pattern: ItemPattern;
}
type Pagination = BaseSelector;
type PaginationRule = BaseSelector;
interface PagePattern {
name: string;
fields?: FieldRule[];
lists?: ListRule[];
pagination?: Pagination;
pagination?: PaginationRule;
}
interface PageData {
data?: Record<string, any>;
list_data?: any[][];
}
type PageData = Record<string, string | number | boolean | PageData[]>;
interface AutoProbeTask extends BaseModel {
autoprobe_id: string;
@@ -73,25 +70,14 @@ export declare global {
usage?: LLMResponseUsage;
}
interface AutoProbeFetchResult extends BaseModel {
autoprobe_id: string;
url: string;
html?: string;
}
type AutoProbeItemType = 'page_pattern' | 'list' | 'field' | 'pagination';
interface AutoProbeNavItem<T = any> extends NavItem<T> {
name?: string;
type?:
| 'page_pattern'
| 'fields'
| 'lists'
| 'pagination'
| 'list'
| 'item'
| 'field';
type?: AutoProbeItemType;
rule?: ListRule | FieldRule | PaginationRule;
children?: AutoProbeNavItem[];
parent?: AutoProbeNavItem;
fieldCount?: number;
field?: FieldRule;
pagination?: Pagination;
}
}

38
frontend/crawlab-ui/src/utils/autoprobe.ts Normal file
View File

@@ -0,0 +1,38 @@
export const getIconBySelectorType = (selectorType: SelectorType): Icon => {
switch (selectorType) {
case 'css':
return ['fab', 'css'];
case 'xpath':
return ['fa', 'code'];
case 'regex':
return ['fa', 'search'];
}
};
export const getIconByExtractType = (extractType?: ExtractType): Icon => {
switch (extractType) {
case 'text':
return ['fa', 'file-alt'];
case 'attribute':
return ['fa', 'tag'];
case 'html':
return ['fa', 'file-code'];
default:
return ['fa', 'question'];
}
};
export const getIconByItemType = (itemType?: AutoProbeItemType): Icon => {
switch (itemType) {
case 'page_pattern':
return ['fa', 'network-wired'];
case 'field':
return ['fa', 'tag'];
case 'list':
return ['fa', 'list'];
case 'pagination':
return ['fa', 'ellipsis-h'];
default:
return ['fa', 'question'];
}
};

4
frontend/crawlab-ui/src/utils/icon.ts
View File

@@ -59,7 +59,7 @@ import {
TAB_NAME_OVERVIEW,
TAB_NAME_PAGES,
TAB_NAME_PATTERNS,
TAB_NAME_PERMISSIONS,
TAB_NAME_PERMISSIONS, TAB_NAME_PREVIEW,
TAB_NAME_RESULTS,
TAB_NAME_ROLES,
TAB_NAME_SCHEDULES,
@@ -130,6 +130,8 @@ export const getIconByTabName = (tabName: string): Icon => {
return ['fa', 'file-alt'];
case TAB_NAME_PATTERNS:
return ['fa', 'network-wired'];
case TAB_NAME_PREVIEW:
return ['fa', 'desktop'];
default:
return ['fa', 'circle'];
}

1
frontend/crawlab-ui/src/utils/index.ts
View File

@@ -30,3 +30,4 @@ export * from './time';
export * from './icon';
export * from './dependency';
export * from './base64';
export * from './autoprobe';

529
frontend/crawlab-ui/src/views/autoprobe/detail/tabs/AutoProbeDetailTabPatterns.vue
View File

@@ -1,10 +1,8 @@
<script setup lang="ts">
import { computed, ref, watch, onMounted } from 'vue';
import { computed, ref, watch } from 'vue';
import { useStore } from 'vuex';
import { plainClone, translate, selectElement } from '@/utils';
import { ElTree, ElInput, ElScrollbar } from 'element-plus';
import type { FilterNodeMethodFunction } from 'element-plus/es/components/tree/src/tree.type';
import { debounce } from 'lodash';
import { getIconByExtractType, getIconByItemType, translate } from '@/utils';
import { cloneDeep } from 'lodash';
// i18n
const t = translate;
@@ -15,355 +13,193 @@ const { autoprobe: state } = store.state as RootStoreState;
// form data
const form = computed<AutoProbe>(() => state.form);
const pageFields = computed(() => form.value?.page_pattern?.fields);
const pageLists = computed(() => form.value?.page_pattern?.lists);
const pagePagination = computed(() => form.value?.page_pattern?.pagination);
const pageData = computed(() => form.value?.page_data || {});
const pageData = computed<PageData>(() => form.value?.page_data || {});
const pageNavItemId = 'page';
const treeRef = ref<InstanceType<typeof ElTree>>();
const searchKeyword = ref('');
const showSearch = ref(false);
const activeNavItem = ref<AutoProbeNavItem>();
const defaultExpandedKeys = ref<string[]>([]);
const normalizeItem = (item: AutoProbeNavItem) => {
const label = item.label ?? `${item.name} (${item.children?.length || 0})`;
let icon: Icon;
if (item.type === 'field') {
const field = item.rule as FieldRule;
icon = getIconByExtractType(field.extraction_type);
} else {
icon = getIconByItemType(item.type);
}
return {
...item,
label,
icon,
} as AutoProbeNavItem;
};
// Helper function to recursively process list items
const processListItem = (list: ListRule): AutoProbeNavItem => {
const children: AutoProbeNavItem[] = [];
const processListItem = (
list: ListRule,
parent?: AutoProbeNavItem
): AutoProbeNavItem => {
const listItem: AutoProbeNavItem = {
id: list.name,
name: list.name,
type: 'list',
rule: list,
children: [],
parent,
};
// Add fields directly if they exist
if (list.item_pattern?.fields && list.item_pattern.fields.length > 0) {
list.item_pattern.fields.forEach((field: FieldRule) => {
children.push({
id: `${list.name}-${field.name}`,
label: field.name,
name: field.name,
icon: ['fa', 'tag'],
type: 'field',
field,
});
listItem.children!.push(
normalizeItem({
id: `${list.name}-${field.name}`,
label: field.name,
name: field.name,
type: 'field',
rule: field,
parent: listItem,
})
);
});
}
// Recursively process nested lists if they exist
if (list.item_pattern?.lists && list.item_pattern.lists.length > 0) {
list.item_pattern.lists.forEach((nestedList: ListRule) => {
children.push(processListItem(nestedList));
listItem.children!.push(processListItem(nestedList, listItem));
});
}
return {
id: list.name,
label: `${list.name} (${children.length})`,
name: list.name,
type: 'list',
icon: ['fa', 'list'],
children,
} as AutoProbeNavItem;
return normalizeItem(listItem);
};
// items
const activeNavItem = ref<AutoProbeNavItem>();
const detailNavItem = computed<AutoProbeNavItem | undefined>(() => {
if (!activeNavItem.value?.type) return;
switch (activeNavItem.value.type) {
case 'page_pattern':
case 'list':
return activeNavItem.value;
case 'field':
case 'pagination':
return activeNavItem.value.parent;
}
});
const computedTreeItems = computed<AutoProbeNavItem[]>(() => {
if (!form.value?.page_pattern) return [];
const children: AutoProbeNavItem[] = [];
const rootItem: AutoProbeNavItem = {
id: pageNavItemId,
name: form.value.page_pattern.name,
type: 'page_pattern',
children: [],
};
// Add fields directly if they exist
if (pageFields.value) {
pageFields.value.forEach(field => {
children.push({
id: field.name,
label: field.name,
name: field.name,
icon: ['fa', 'tag'],
type: 'field',
field,
});
rootItem.children!.push(
normalizeItem({
id: field.name,
label: field.name,
name: field.name,
type: 'field',
rule: field,
parent: rootItem,
})
);
});
}
// Add lists directly if they exist
if (pageLists.value) {
pageLists.value.forEach(list => {
children.push(processListItem(list));
rootItem.children!.push(processListItem(list, rootItem));
});
}
// Add pagination if it exists
if (pagePagination.value) {
children.push({
id: 'pagination',
label: t('components.autoprobe.navItems.pagination'),
name: t('components.autoprobe.navItems.pagination'),
type: 'pagination',
icon: ['fa', 'ellipsis-h'],
pagination: pagePagination.value,
});
rootItem.children!.push(
normalizeItem({
id: 'pagination',
label: t('components.autoprobe.navItems.pagination'),
name: t('components.autoprobe.navItems.pagination'),
type: 'pagination',
rule: pagePagination.value,
parent: rootItem,
})
);
}
return [
{
id: 'page',
label: `${form.value.page_pattern.name} (${children.length})`,
name: form.value.page_pattern.name,
type: 'page_pattern',
icon: ['fa', 'network-wired'],
children,
},
];
return [normalizeItem(rootItem)];
});
const treeItems = ref<AutoProbeNavItem[]>([]);
watch(
() => state.form,
() => {
treeItems.value = plainClone(computedTreeItems.value);
}
treeItems.value = cloneDeep(computedTreeItems.value);
},
{ immediate: true }
);
// Function to get field data from page data
const getFieldData = (fieldName: string) => {
if (!pageData.value) return undefined;
// Use type assertion to treat pageData as a record with string keys
return (pageData.value as Record<string, any>)[fieldName];
// ref
const sidebarRef = ref();
const detailContainerRef = ref<HTMLElement | null>(null);
const onNodeSelect = (item: AutoProbeNavItem) => {
activeNavItem.value = item;
};
// Function to get data for a specific navigation item
const getNavItemData = (item: AutoProbeNavItem) => {
if (!pageData.value) return undefined;
// Convert to Record<string, any> to handle dynamic property access
const data = pageData.value as Record<string, any>;
switch (item.type) {
case 'field':
// For fields, extract from page data by field name
return getFieldData(item.id);
case 'list':
// For lists, get the corresponding array in page data
return data[item.id];
case 'pagination':
// For pagination, extract pagination-related data
return data.pagination;
case 'page_pattern':
// Return all page data for page pattern
return pageData.value;
default:
return undefined;
}
const onItemRowClick = (id: string) => {
const item = sidebarRef.value?.getNode(id);
if (!item) return;
activeNavItem.value = sidebarRef.value?.getNode(id);
};
// Context menu functionality
const activeContextMenuNavItem = ref<AutoProbeNavItem>();
const contextMenuVisibleMap = ref<Record<string, boolean>>({});
const isContextMenuVisible = (id: string) => {
if (!contextMenuItems.value?.length) return false;
if (!activeContextMenuNavItem.value) return false;
if (activeContextMenuNavItem.value?.id !== id) return false;
return contextMenuVisibleMap.value[id] || false;
// Handle results container resize
const onSizeChange = (size: number) => {
if (!detailContainerRef.value) return;
detailContainerRef.value.style.flex = `0 0 calc(100% - ${size}px)`;
detailContainerRef.value.style.height = `calc(100% - ${size}px)`;
};
const onActionsClick = (item: AutoProbeNavItem) => {
activeContextMenuNavItem.value = item;
contextMenuVisibleMap.value[item.id] = true;
};
const onContextMenuHide = (id: string) => {
activeContextMenuNavItem.value = undefined;
contextMenuVisibleMap.value[id] = false;
};
const contextMenuItems = computed(() => {
if (!activeContextMenuNavItem.value) return [];
const { id, type } = activeContextMenuNavItem.value;
if (!contextMenuVisibleMap.value[id]) return [];
switch (type) {
case 'field':
return [
{
title: t('common.actions.view'),
icon: ['fa', 'eye'],
action: () =>
selectNode(activeContextMenuNavItem.value as AutoProbeNavItem),
},
];
case 'list':
return [
{
title: t('common.actions.view'),
icon: ['fa', 'eye'],
action: () =>
selectNode(activeContextMenuNavItem.value as AutoProbeNavItem),
},
];
default:
return [];
}
});
// Search functionality
const onSearchFilter: FilterNodeMethodFunction = (value, data) => {
if (!value) return true;
return data.label.toLowerCase().includes(value.toLowerCase());
};
const debouncedFilter = debounce(() => {
treeRef.value?.filter(searchKeyword.value);
}, 300);
watch(searchKeyword, debouncedFilter);
// Node selection and expansion tracking
const onNodeClick = async (data: AutoProbeNavItem) => {
await selectNode(data);
};
const selectNode = async (data: AutoProbeNavItem) => {
const { id } = data;
activeNavItem.value = data;
// Display content in the right panel based on selected node
// (Implementation would depend on what content you want to show)
// Highlight current node
setTimeout(() => {
treeRef.value?.setCurrentKey(id);
}, 0);
};
const onNodeExpand = (data: AutoProbeNavItem) => {
defaultExpandedKeys.value.push(data.id);
};
const onNodeCollapse = (data: AutoProbeNavItem) => {
const idx = defaultExpandedKeys.value.findIndex(id => id === data.id);
defaultExpandedKeys.value.splice(idx, 1);
};
const onContextMenuClick = (event: MouseEvent, data: AutoProbeNavItem) => {
event.stopPropagation();
activeContextMenuNavItem.value = data;
contextMenuVisibleMap.value[data.id] = true;
};
const onSearchClick = () => {
showSearch.value = !showSearch.value;
};
const onRefresh = () => {
// Refresh data if needed
treeItems.value = plainClone(computedTreeItems.value);
};
// Sidebar resizing
const widthKey = ref('autoprobe.sidebar.width');
const sidebarRef = ref<HTMLElement | null>(null);
onMounted(() => {
treeItems.value = plainClone(computedTreeItems.value);
});
defineOptions({ name: 'ClAutoProbeDetailTabPatterns' });
</script>
<template>
<div class="autoprobe-detail-tab-patterns">
<div ref="sidebarRef" class="sidebar">
<cl-resize-handle :target-ref="sidebarRef" :size-key="widthKey" />
<div class="sidebar-actions">
<cl-icon :icon="['fa', 'refresh']" @click="onRefresh" />
<cl-icon
:class="showSearch ? 'selected' : ''"
:icon="['fa', 'search']"
@click="onSearchClick"
/>
</div>
<div v-if="showSearch" class="sidebar-search">
<el-input
v-model="searchKeyword"
:placeholder="t('common.search.placeholder')"
clearable
@clear="
() => {
searchKeyword = '';
showSearch = false;
}
"
/>
</div>
<el-scrollbar>
<el-tree
ref="treeRef"
node-key="id"
:data="treeItems"
:filter-node-method="onSearchFilter"
:expand-on-click-node="false"
:default-expanded-keys="defaultExpandedKeys"
highlight-current
@node-click="onNodeClick"
@node-contextmenu="onContextMenuClick"
@node-expand="onNodeExpand"
@node-collapse="onNodeCollapse"
>
<template #default="{ data }">
<cl-context-menu
:visible="isContextMenuVisible(data.id)"
:style="{ flex: 1, paddingRight: '5px' }"
>
<template #reference>
<div class="node-wrapper" :title="data.label">
<span class="icon-wrapper">
<cl-icon
v-if="data.loading"
:icon="['fa', 'spinner']"
spinning
/>
<cl-icon v-else :icon="data.icon || ['fa', 'folder']" />
</span>
<span class="label">
{{ data.label }}
</span>
</div>
<div class="actions">
<cl-icon
class="more"
:icon="['fa', 'ellipsis']"
@click.stop="onActionsClick(data)"
/>
</div>
</template>
<cl-context-menu-list
v-if="isContextMenuVisible(data.id)"
:items="contextMenuItems"
@hide="onContextMenuHide(data.id)"
/>
</cl-context-menu>
</template>
</el-tree>
</el-scrollbar>
</div>
<cl-auto-probe-page-patterns-sidebar
ref="sidebarRef"
:active-nav-item-id="activeNavItem?.id"
:tree-items="treeItems"
:default-expanded-keys="[pageNavItemId]"
@node-select="onNodeSelect"
/>
<div class="content">
<template v-if="activeNavItem?.type === 'field'">
<cl-auto-probe-field-detail
:field="activeNavItem"
:page-data="getNavItemData(activeNavItem)"
/>
</template>
<template v-else-if="activeNavItem?.type === 'list'">
<cl-auto-probe-list-detail
:list="activeNavItem"
:page-data="getNavItemData(activeNavItem)"
/>
</template>
<template v-else-if="activeNavItem?.type === 'pagination'">
<cl-auto-probe-pagination-detail
:pagination="activeNavItem"
:page-data="getNavItemData(activeNavItem)"
/>
</template>
<template v-else-if="activeNavItem?.type === 'page_pattern'">
<cl-auto-probe-page-pattern-detail :page-pattern="activeNavItem" />
</template>
<div v-else class="placeholder">
{{
t('components.autoprobe.patterns.selectItem') ||
'Select an item to view details'
}}
<div ref="detailContainerRef" class="detail-container">
<template v-if="detailNavItem">
<cl-auto-probe-item-detail
:item="detailNavItem"
:active-id="activeNavItem?.id"
@row-click="onItemRowClick"
/>
</template>
<div v-else class="placeholder">
{{ t('components.autoprobe.patterns.selectItem') }}
</div>
</div>
<!--TODO: implement the data for activeNavItem-->
<cl-auto-probe-results-container
v-if="detailNavItem"
:data="pageData"
:fields="activeNavItem?.children"
@size-change="onSizeChange"
/>
</div>
</div>
</template>
@@ -373,108 +209,15 @@ defineOptions({ name: 'ClAutoProbeDetailTabPatterns' });
height: 100%;
display: flex;
.sidebar {
flex: 0 0 240px;
height: 100%;
overflow: hidden;
border-right: 1px solid var(--el-border-color);
display: flex;
flex-direction: column;
position: relative;
.sidebar-actions {
height: 41px;
flex: 0 0 41px;
padding: 5px;
display: flex;
align-items: center;
gap: 5px;
color: var(--cl-primary-color);
border-bottom: 1px solid var(--el-border-color);
& > * {
display: flex;
align-items: center;
}
&:deep(.icon) {
cursor: pointer;
padding: 6px;
font-size: 14px;
width: 14px;
height: 14px;
border-radius: 50%;
}
&:deep(.icon.selected),
&:deep(.icon:hover) {
background-color: var(--cl-primary-plain-color);
}
}
.sidebar-search {
height: 38px;
flex: 0 0 38px;
border-bottom: 1px solid var(--el-border-color);
&:deep(.el-input .el-input__wrapper) {
box-shadow: none;
border: none;
}
}
.el-tree {
min-width: fit-content;
&:deep(.el-tree-node__content:hover .actions .icon) {
display: flex !important;
}
&:deep(.el-tree-node__content) {
width: 100%;
position: relative;
.node-wrapper {
display: flex;
align-items: center;
position: relative;
width: 100%;
.icon-wrapper {
width: 20px;
display: flex;
}
.label {
flex: 0 0 auto;
}
}
.actions {
display: flex;
gap: 5px;
position: absolute;
top: 0;
right: 5px;
height: 100%;
align-items: center;
&:deep(.icon.more) {
display: none;
}
}
}
}
}
.content {
flex: 1;
overflow: auto;
display: flex;
flex-direction: column;
overflow: hidden;
.detail-panel {
border: 1px solid var(--el-border-color);
border-radius: 4px;
padding: 16px;
.detail-container {
flex: 1;
overflow: auto;
}
.placeholder {