Merge pull request #1035 from crawlab-team/develop

Develop
2026-01-21 17:21:09 +01:00 · 2021-12-24 18:06:19 +08:00
parent aea571dcf9 6c1f78b114
commit e4b1bb5f70
14 changed files with 131 additions and 19 deletions
--- a/2
+++ b/2
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux

 # install backend
 RUN pip install scrapy pymongo bs4 requests
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500

 # add files
 COPY ./backend/conf /app/backend/conf
--- a/Dockerfile.cn
+++ b/Dockerfile.cn
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux

 # install backend
 RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500

 # add files
 COPY ./backend/conf /app/backend/conf
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux

 # install backend
 RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500

 # add files
 COPY ./backend/conf /app/backend/conf
--- a/backend/cmd/root.go
+++ b/backend/cmd/root.go
@@ -2,6 +2,7 @@ package cmd

 import (
 	"fmt"
+	"github.com/apex/log"
 	"strings"

 	"github.com/spf13/cobra"
@@ -54,4 +55,17 @@ func initConfig() {
 	if err := viper.ReadInConfig(); err == nil {
 		fmt.Println("Using config file:", viper.ConfigFileUsed())
 	}
+
+	// initialize log level
+	initLogLevel()
+}
+
+func initLogLevel() {
+	// set log level
+	logLevel := viper.GetString("log.level")
+	l, err := log.ParseLevel(logLevel)
+	if err != nil {
+		l = log.InfoLevel
+	}
+	log.SetLevel(l)
 }
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -4,7 +4,7 @@ go 1.16

 require (
 	github.com/apex/log v1.9.0
-	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718
 	github.com/crawlab-team/go-trace v0.1.1
 	github.com/gin-gonic/gin v1.7.1
 	github.com/spf13/cobra v1.1.3
--- a/backend/go.mod.local
+++ b/backend/go.mod.local
@@ -11,11 +11,10 @@ replace (

 require (
 	github.com/apex/log v1.9.0
-	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211213.2006
-	github.com/crawlab-team/go-trace v0.1.0
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134
+	github.com/crawlab-team/go-trace v0.1.1
 	github.com/gin-gonic/gin v1.7.1
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/viper v1.7.1
-	go.mongodb.org/mongo-driver v1.6.0 // indirect
 	go.uber.org/dig v1.10.0
 )
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -63,8 +63,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134 h1:J2NKuIhkXZPlfx9hHT2L4VLS9FMHzEaDGgHP/8ZJKkI=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718 h1:nNmFoR/o+xEZ6ywe9mwx0QroF+QSHrJR5DotOm/r/tg=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY=
 github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
 github.com/crawlab-team/crawlab-db v0.1.3 h1:RqLoXGZEMUH1B8SQB5OcNmJeyY2xILvwyhv4X9faWl4=
 github.com/crawlab-team/crawlab-db v0.1.3/go.mod h1:kPkGZ1P802XdbFFb8byMpZfNG2lWTNoWNRy4beS0/QY=
--- a/backend/test/config-worker-03.json
+++ b/backend/test/config-worker-03.json
@@ -0,0 +1,10 @@
+{
+  "key": "worker-03",
+  "is_master": false,
+  "name": "Worker Node 03",
+  "ip": "",
+  "mac": "",
+  "hostname": "",
+  "description": "",
+  "auth_key": "Crawlab2021!"
+}
--- a/changelog/v0.6.0-beta.20211224-zh.md
+++ b/changelog/v0.6.0-beta.20211224-zh.md
@@ -0,0 +1,32 @@
+# 更新日志 (v0.6.0-beta.20211224)
+
+## 概览
+
+这是下个重大版本 v0.6.0 的第三次 beta 发布。伴随着更多功能和优化，正式版本 v0.6.0 将会很快发布了。
+
+## 升级优化
+
+- [x] **国际化**. 支持中文.
+- [x] **命令行工具上传爬虫**. [#1020](https://github.com/crawlab-team/crawlab/issues/1020)
+- [x] **官方插件**. 允许用户在 Crawlab 用户界面上安装 [官方插件](https://docs-next.crawlab.cn/zh/guide/use-crawlab/public-plugins/).
+- [x] **更多文档**. 加入了关于插件和命令行工具的文档.
+
+## Bug 修复
+
+- [x] **上传文件问题** [#990](https://github.com/crawlab-team/crawlab/issues/990) [#1014](https://github.com/crawlab-team/crawlab/issues/1014) [#1022](https://github.com/crawlab-team/crawlab/issues/1022)
+- [x] **无法重命名文件** [#1008](https://github.com/crawlab-team/crawlab/issues/1008)
+
+## TODOs
+- [ ] **关联任务**. 如果任务执行模式为 “所有节点” 或 “指定节点”，那么将会有主任务和子任务之分。
+- [ ] **Crontab 编辑器**. 可视化 Crontab 编辑的前端组件。
+- [ ] **结果去重**.
+- [ ] **环境变量**.
+- [ ] **前端易用性优化**. 更多高级功能，例如表格形式保存。
+- [ ] **日志自动清理**.
+- [ ] **跟多文档**.
+- [ ] **前端输出文件大小优化**.
+
+## 未来计划
+
+下一个版本有可能是 v0.6.0 的正式版本，但是没有确定。本次发布版本将会进行更多的测试，以保证其健壮性以及生产可用。
+
--- a/changelog/v0.6.0-beta.20211224.md
+++ b/changelog/v0.6.0-beta.20211224.md
@@ -0,0 +1,32 @@
+# Change Log (v0.6.0-beta.20211224)
+
+## Overview
+
+This is the third beta release for the next major version v0.6.0. With more features and optimization coming in, the release of official version v0.6.0 is approaching soon. 
+
+## Enhancement
+
+- [x] **Internationalization**. Support Chinese.
+- [x] **CLI Upload Spider**. [#1020](https://github.com/crawlab-team/crawlab/issues/1020)
+- [x] **Official Plugins**. Allow users to install [official plugins](https://docs-next.crawlab.cn/en/guide/use-crawlab/public-plugins/) on Crawlab web UI.
+- [x] **More Documentation**. Added documentation for plugins and CLI.
+
+## Bug Fixes
+
+- [x] **Upload File Issue** [#990](https://github.com/crawlab-team/crawlab/issues/990) [#1014](https://github.com/crawlab-team/crawlab/issues/1014) [#1022](https://github.com/crawlab-team/crawlab/issues/1022)
+- [x] **Unable to Rename File** [#1008](https://github.com/crawlab-team/crawlab/issues/1008)
+
+## TODOs
+- [ ] **Associated Tasks**. There will be main tasks and their sub-tasks if task mode is "all nodes" or "selected nodes".
+- [ ] **Crontab Editor**. Frontend component that visualize the crontab editing.
+- [ ] **Results Deduplication**.
+- [ ] **Environment Variables**.
+- [ ] **Frontend Utility Enhancement**. Advanced features such as saved table customization. 
+- [ ] **Log Auto Cleanup**.
+- [ ] **More Documentation**. 
+- [ ] **E2E Tests**. 
+- [ ] **Frontend Output File Size Optimization**.
+
+## What Next
+
+The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robustness and production-ready deployment.
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
  "name": "@crawlab/app",
-  "version": "0.6.0-beta.20211221.1200",
+  "version": "0.6.0-beta.20211224.1718",
  "description": "",
  "scripts": {
    "serve": "vue-cli-service serve",
@@ -15,7 +15,7 @@
  },
  "license": "BSD-3-Clause",
  "dependencies": {
-    "crawlab-ui": "0.6.0-beta.20211219.1200",
+    "crawlab-ui": "0.6.0-beta.20211224.1718",
    "vue": "3.0.11",
    "vue-router": "^4.0.11"
  },
--- a/frontend/public/index.html
+++ b/frontend/public/index.html
@@ -1,12 +1,12 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
+  <title><%= htmlWebpackPlugin.options.title %></title>
+  <link rel="icon" type="image/x-icon" href="<%= BASE_URL %>favicon.ico">
+  <link rel="stylesheet" href="font-awesome.min.css">
  <meta charset="utf-8">
  <meta content="IE=edge" http-equiv="X-UA-Compatible">
  <meta content="width=device-width,initial-scale=1.0" name="viewport">
-  <link href="<%= BASE_URL %>favicon.ico" rel="icon">
-  <link href="font-awesome.min.css" rel="stylesheet">
-  <title><%= htmlWebpackPlugin.options.title %></title>
  <script>
  window.VUE_APP_API_BASE_URL = '<%= VUE_APP_API_BASE_URL %>'
  </script>
--- a/nginx/crawlab.conf
+++ b/nginx/crawlab.conf
@@ -3,7 +3,7 @@ server {
 	gzip_min_length 1k;
 	gzip_buffers 4 16k;
 	gzip_comp_level 2;
-	gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
+	gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png image/x-icon;
 	gzip_vary off;
 	gzip_disable "MSIE [1-6]\.";
 	client_max_body_size 200m;
--- a/workspace/docker-compose.yml
+++ b/workspace/docker-compose.yml
@@ -1,10 +1,24 @@
 version: '3.3'
 services:
+  build:
+    build:
+      context: ./dockerfiles/golang
+    command: /bin/bash -c "rm -f /backend/dist/crawlab && go build -o ./dist/crawlab ./"
+    volumes:
+      - ./dist:/backend/dist
+      - ./.crawlab/go/pkg/mod:/go/pkg/mod
+      - ../backend:/backend
+      - ../backend/go.mod.local:/backend/go.mod
+      - ../..:/libs/crawlab-team
+    environment:
+      GOPROXY: https://goproxy.cn,direct
+
  master:
    build:
      context: ./dockerfiles/golang
-    command: /bin/bash -c "/app/bin/docker-start-master.sh && air -c .air.master.conf"
+    command: /bin/bash -c "/app/bin/docker-start-master.sh && /dist/crawlab master"
    volumes:
+      - ./dist:/dist
      - ./.crawlab/master:/root/.crawlab
      - ./.crawlab/go/pkg/mod:/go/pkg/mod
      - ../backend:/backend
@@ -17,22 +31,29 @@ services:
      CRAWLAB_NODE_MASTER: "Y"
      CRAWLAB_NODE_NAME: "Master Node"
      CRAWLAB_MONGO_HOST: "mongo"
+      CRAWLAB_LOG_LEVEL: debug
+      GOPROXY: https://goproxy.cn,direct
    ports:
      - "9080:8080"
      - "9000:8000"
+      - "9866:9666"
+      - "9888:8888"
    depends_on:
      - mongo

  worker01:
    build:
      context: ./dockerfiles/golang
-    command: "air -c .air.worker.conf"
+    command: /bin/bash -c "/dist/crawlab worker"
    environment:
      CRAWLAB_NODE_MASTER: "N"
      CRAWLAB_NODE_NAME: "Worker Node 01"
      CRAWLAB_GRPC_ADDRESS: "master"
      CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
+      CRAWLAB_LOG_LEVEL: debug
+      GOPROXY: https://goproxy.cn,direct
    volumes:
+      - ./dist:/dist
      - ./.crawlab/worker01:/root/.crawlab
      - ./.crawlab/go/pkg/mod:/go/pkg/mod
      - ../backend:/backend
@@ -44,13 +65,16 @@ services:
  worker02:
    build:
      context: ./dockerfiles/golang
-    command: "air -c .air.worker.conf"
+    command: /bin/bash -c "/dist/crawlab worker"
    environment:
      CRAWLAB_NODE_MASTER: "N"
-      CRAWLAB_NODE_NAME: "Worker Node 01"
+      CRAWLAB_NODE_NAME: "Worker Node 02"
      CRAWLAB_GRPC_ADDRESS: "master"
      CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
+      CRAWLAB_LOG_LEVEL: debug
+      GOPROXY: https://goproxy.cn,direct
    volumes:
+      - ./dist:/dist
      - ./.crawlab/worker02:/root/.crawlab
      - ./.crawlab/go/pkg/mod:/go/pkg/mod
      - ../backend:/backend
@@ -58,6 +82,7 @@ services:
      - ../..:/libs/crawlab-team
    depends_on:
      - master
+      - worker01

  mongo:
    image: mongo:4