diff --git a/core/utils/filter.go b/core/utils/filter.go index 1884de3d..697cc8a3 100644 --- a/core/utils/filter.go +++ b/core/utils/filter.go @@ -8,11 +8,12 @@ import ( // FilterToQuery Translate entity.Filter to bson.M func FilterToQuery(f interfaces.Filter) (q bson.M) { + q = bson.M{} + if f == nil || f.IsNil() { - return nil + return q } - q = bson.M{} for _, cond := range f.GetConditions() { key := cond.GetKey() op := cond.GetOp() @@ -44,8 +45,6 @@ func FilterToQuery(f interfaces.Filter) (q bson.M) { // ignore invalid operation } } - if len(q) == 0 { - return nil - } + return q } diff --git a/mcp/package.json b/mcp/package.json index b84c3609..27c4fb6f 100644 --- a/mcp/package.json +++ b/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@crawlab/mcp", - "version": "0.1.0", + "version": "0.1.1-dev.6", "description": "MCP server for interacting with Crawlab web crawler management platform", "license": "MIT", "author": "Crawlab Team", @@ -24,7 +24,15 @@ "lint": "eslint src/**/*.ts", "lint:fix": "eslint src/**/*.ts --fix", "clean": "shx rm -rf dist", - "test": "jest --passWithNoTests" + "test": "jest --passWithNoTests", + "version:dev": "npm version prerelease --preid=dev", + "version:patch": "npm version patch", + "version:minor": "npm version minor", + "version:major": "npm version major", + "publish:dev": "npm run version:dev && npm publish --tag dev", + "publish:latest": "npm publish", + "release:dev": "npm run build && npm run publish:dev", + "release:prod": "npm run build && npm run version:patch && npm run publish:latest" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.12.2", @@ -46,5 +54,5 @@ "ts-jest": "^29.4.0", "typescript": "^5.8.3" }, - "packageManager": "pnpm@10.12.1" + "packageManager": "pnpm@9.9.0+sha512.60c18acd138bff695d339be6ad13f7e936eea6745660d4cc4a776d5247c540d0edee1a563695c183a66eb917ef88f2b4feb1fc25f32a7adcadc7aaf3438e99c1" } diff --git a/mcp/pnpm-lock.yaml b/mcp/pnpm-lock.yaml index cae5c1d0..63cd9507 100644 --- a/mcp/pnpm-lock.yaml +++ b/mcp/pnpm-lock.yaml @@ -1050,8 +1050,8 @@ packages: resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} engines: {node: '>=10'} - caniuse-lite@1.0.30001723: - resolution: {integrity: sha512-1R/elMjtehrFejxwmexeXAtae5UO9iSyFn6G/I806CYC/BLyyBk1EPhrKBkWhy6wM6Xnm47dSJQec+tLJ39WHw==} + caniuse-lite@1.0.30001724: + resolution: {integrity: sha512-WqJo7p0TbHDOythNTqYujmaJTvtYRZrjpP8TCvH6Vb9CYJerJNKamKzIWOM4BkQatWj9H2lYulpdAQNBe7QhNA==} chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} @@ -1233,8 +1233,8 @@ packages: engines: {node: '>=0.10.0'} hasBin: true - electron-to-chromium@1.5.170: - resolution: {integrity: sha512-GP+M7aeluQo9uAyiTCxgIj/j+PrWhMlY7LFVj8prlsPljd0Fdg9AprlfUi+OCSFWy9Y5/2D/Jrj9HS8Z4rpKWA==} + electron-to-chromium@1.5.171: + resolution: {integrity: sha512-scWpzXEJEMrGJa4Y6m/tVotb0WuvNmasv3wWVzUAeCgKU0ToFOhUW6Z+xWnRQANMYGxN4ngJXIThgBJOqzVPCQ==} emittery@0.13.1: resolution: {integrity: sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==} @@ -3791,8 +3791,8 @@ snapshots: browserslist@4.25.0: dependencies: - caniuse-lite: 1.0.30001723 - electron-to-chromium: 1.5.170 + caniuse-lite: 1.0.30001724 + electron-to-chromium: 1.5.171 node-releases: 2.0.19 update-browserslist-db: 1.1.3(browserslist@4.25.0) @@ -3830,7 +3830,7 @@ snapshots: camelcase@6.3.0: {} - caniuse-lite@1.0.30001723: {} + caniuse-lite@1.0.30001724: {} chalk@4.1.2: dependencies: @@ -3992,7 +3992,7 @@ snapshots: dependencies: jake: 10.9.2 - electron-to-chromium@1.5.170: {} + electron-to-chromium@1.5.171: {} emittery@0.13.1: {} diff --git a/mcp/src/client.ts b/mcp/src/client.ts index 98680ba4..529078f2 100644 --- a/mcp/src/client.ts +++ b/mcp/src/client.ts @@ -185,11 +185,16 @@ export class CrawlabClient { private client: AxiosInstance; private baseURL: string; - constructor(baseURL: string, apiToken?: string, timeout: number = 30000) { - this.baseURL = baseURL.replace(/\/$/, ''); // Remove trailing slash + constructor(apiUrl: string, apiToken?: string, timeout: number = 30000) { + this.baseURL = apiUrl.replace(/\/$/, ''); // Remove trailing slash + + // Warn if no API token is provided + if (!apiToken) { + console.error('ℹ️ INFO: No API token provided - some endpoints may require authentication'); + } this.client = axios.create({ - baseURL: `${this.baseURL}/api`, + baseURL: this.baseURL, timeout, headers: { 'Content-Type': 'application/json', diff --git a/mcp/src/index.ts b/mcp/src/index.ts index bb36f66e..7f360056 100644 --- a/mcp/src/index.ts +++ b/mcp/src/index.ts @@ -2,24 +2,49 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; -import { configurePrompts } from './prompts'; -import { configureAllTools } from './tools'; -import { CrawlabClient } from './client'; -import { packageVersion } from './version'; +import { configurePrompts } from './prompts.js'; +import { configureAllTools } from './tools.js'; +import { CrawlabClient } from './client.js'; +import { packageVersion } from './version.js'; const args = process.argv.slice(2); if (args.length < 1) { - console.error('Usage: mcp-server-crawlab [api_token]'); - console.error('Example: mcp-server-crawlab http://localhost:8080'); + console.error('Usage: mcp-server-crawlab [api_token]'); + console.error('Example (Docker): mcp-server-crawlab http://localhost:8080/api'); + console.error('Example (Local dev): mcp-server-crawlab http://localhost:8000'); console.error( - 'Example: mcp-server-crawlab http://localhost:8080 your-api-token' + 'Example with token: mcp-server-crawlab http://localhost:8080/api your-api-token' ); process.exit(1); } -const crawlabUrl = args[0]; +const crawlabApiEndpoint = args[0] || process.env.CRAWLAB_API_URL || process.env.CRAWLAB_API_ENDPOINT; const apiToken = args[1] || process.env.CRAWLAB_API_TOKEN; +// Check for missing API URL +if (!crawlabApiEndpoint) { + console.error('❌ ERROR: Crawlab API URL is required!'); + console.error(' Provide API URL via:'); + console.error(' 1. Command argument: mcp-server-crawlab [token]'); + console.error(' 2. Environment variable: export CRAWLAB_API_ENDPOINT=http://localhost:8080/api'); + console.error(' Examples:'); + console.error(' - Docker: http://localhost:8080/api'); + console.error(' - Local dev: http://localhost:8000'); + console.error(''); + process.exit(1); +} + +// Warning if API token is missing +if (!apiToken) { + console.error('⚠️ WARNING: CRAWLAB_API_TOKEN is not set!'); + console.error(' This may cause authentication issues with the Crawlab API.'); + console.error(' To fix this:'); + console.error(' 1. Set environment variable: export CRAWLAB_API_TOKEN=your_token'); + console.error(' 2. Or pass token as argument: mcp-server-crawlab '); + console.error(' 3. Or add to your shell profile (~/.zshrc, ~/.bashrc)'); + console.error(''); +} + async function main() { const server = new McpServer({ name: 'Crawlab MCP Server', @@ -27,15 +52,16 @@ async function main() { }); // Initialize Crawlab client - const client = new CrawlabClient(crawlabUrl, apiToken); + const client = new CrawlabClient(crawlabApiEndpoint!, apiToken); // Configure prompts and tools configurePrompts(server); configureAllTools(server, client); const transport = new StdioServerTransport(); - console.error(`Crawlab MCP Server version: ${packageVersion}`); - console.error(`Connecting to Crawlab at: ${crawlabUrl}`); + console.info(`Crawlab MCP Server version: ${packageVersion}`); + console.info(`Crawlab API endpoint: ${crawlabApiEndpoint}`); + console.info(`API token: ${apiToken ? `${apiToken.substring(0, 8)}...` : 'Not provided'}`); await server.connect(transport); } diff --git a/mcp/src/tools.ts b/mcp/src/tools.ts index de4cd82e..a9dd7933 100644 --- a/mcp/src/tools.ts +++ b/mcp/src/tools.ts @@ -1,16 +1,16 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { CrawlabClient } from "./client.js"; -import { configureSpiderTools } from '@tools/spiders'; -import { configureTaskTools } from '@tools/tasks'; -import { configureNodeTools } from '@tools/nodes'; -import { configureScheduleTools } from '@tools/schedules'; -import { configureSystemTools } from '@tools/system'; -import { configureProjectTools } from '@tools/projects'; -import { configureDatabaseTools } from '@tools/databases'; -import { configureGitTools } from '@tools/git'; -import { configureStatsTools } from '@tools/stats'; -import { configureAITools } from '@tools/ai'; +import { configureSpiderTools } from './tools/spiders.js'; +import { configureTaskTools } from './tools/tasks.js'; +import { configureNodeTools } from './tools/nodes.js'; +import { configureScheduleTools } from './tools/schedules.js'; +import { configureSystemTools } from './tools/system.js'; +import { configureProjectTools } from './tools/projects.js'; +import { configureDatabaseTools } from './tools/databases.js'; +import { configureGitTools } from './tools/git.js'; +import { configureStatsTools } from './tools/stats.js'; +import { configureAITools } from './tools/ai.js'; export function configureAllTools(server: McpServer, client: CrawlabClient) { configureSpiderTools(server, client); diff --git a/mcp/src/version.ts b/mcp/src/version.ts index 48b8fe64..693757d5 100644 --- a/mcp/src/version.ts +++ b/mcp/src/version.ts @@ -1 +1 @@ -export const packageVersion = "0.1.0"; +export const packageVersion = "0.1.1-dev.6";