Merge pull request #1067 from crawlab-team/develop

Develop

DISCLAIMER.md

@@ -8,5 +8,5 @@ This Disclaimer and privacy protection statement (hereinafter referred to as "di
 4. The user shall bear the risk of using crawlab by himself, we do not make any form of guarantee, and we will not bear any legal responsibility for the user's failure to upgrade and update normally due to any technical reasons such as network condition and communication line.
 5. When users use crawlab to grab the target website, they need to comply with the laws and regulations related to crawlers, such as the network security law. Do not collect personal information of citizens without authorization, cause the target website to be paralyzed by DDoS, or fail to comply with the robots.txt protocol and other illegal means of the target website.
 6. Crawlab respects and protects the personal privacy of all users and will not steal any information from users' computers.
-7. Copyright of the system: the crawleb development team owns the intellectual property rights, copyrights, copyrights and use rights for all developed or jointly developed products, which are protected by applicable intellectual property rights, copyrights, trademarks, service trademarks, patents or other laws.
-8. Communication: any company or individual who publishes or disseminates our software on the Internet is allowed, but the crawlab development team shall not be responsible for any legal and criminal events that may be caused by the company or individual disseminating the software.
+7. Copyright of the system: Crawlab development team owns the intellectual property rights, copyrights, copyrights and use rights for all developed or jointly developed products, which are protected by applicable intellectual property rights, copyrights, trademarks, service trademarks, patents or other laws.
+8. Communication: any company or individual who publishes or disseminates our software on the Internet is allowed, but the crawlab development team shall not be responsible for any legal and criminal events that may be caused by the company or individual disseminating the software.

backend/.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = tab
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[{*.yml, *.yaml, *.json}]
+indent_size = 2

backend/Makefile

@@ -1,32 +0,0 @@
-SHELL := /bin/bash
-BASEDIR = $(shell pwd)
-
-# build with verison infos
-versionDir = "apiserver/pkg/version"
-gitTag = $(shell if [ "`git describe --tags --abbrev=0 2>/dev/null`" != "" ];then git describe --tags --abbrev=0; else git log --pretty=format:'%h' -n 1; fi)
-buildDate = $(shell TZ=Asia/Shanghai date +%FT%T%z)
-gitCommit = $(shell git log --pretty=format:'%H' -n 1)
-gitTreeState = $(shell if git status|grep -q 'clean';then echo clean; else echo dirty; fi)
-
-ldflags="-w -X ${versionDir}.gitTag=${gitTag} -X ${versionDir}.buildDate=${buildDate} -X ${versionDir}.gitCommit=${gitCommit} -X ${versionDir}.gitTreeState=${gitTreeState}"
-
-all: gotool
-	@go build -v -ldflags ${ldflags} .
-clean:
-	rm -f apiserver
-	find . -name "[._]*.s[a-w][a-z]" | xargs -i rm -f {}
-gotool:
-	gofmt -w .
-	go tool vet . |& grep -v vendor;true
-ca:
-	openssl req -new -nodes -x509 -out conf/server.crt -keyout conf/server.key -days 3650 -subj "/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=127.0.0.1/emailAddress=xxxxx@qq.com"
-
-help:
-	@echo "make - compile the source code"
-	@echo "make clean - remove binary file and vim swp files"
-	@echo "make gotool - run go tool 'fmt' and 'vet'"
-	@echo "make ca - generate ca files"
-
-.PHONY: clean gotool ca help
-
-

backend/Taskfile.yml

@@ -0,0 +1,12 @@
+version: '3'
+
+tasks:
+  dev:
+    desc: Switch to dev mode for local development.
+    cmds:
+      - cp -f go.mod.dev go.mod
+
+  deploy:
+    desc: Switch to deploy mode for code publish.
+    cmds:
+      - echo 'not implemented'

backend/go.mod

@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/apex/log v1.9.0
-	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20220128.1200
 	github.com/crawlab-team/go-trace v0.1.1
 	github.com/gin-gonic/gin v1.7.1
 	github.com/spf13/cobra v1.1.3

backend/go.mod.dev

@@ -0,0 +1,20 @@
+module crawlab
+
+go 1.16
+
+replace (
+    github.com/crawlab-team/crawlab-core => ../../crawlab-core
+    github.com/crawlab-team/crawlab-vcs => ../../crawlab-vcs
+    github.com/crawlab-team/crawlab-fs => ../../crawlab-fs
+    github.com/crawlab-team/crawlab-db => ../../crawlab-db
+)
+
+require (
+	github.com/apex/log v1.9.0
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200
+	github.com/crawlab-team/go-trace v0.1.1
+	github.com/gin-gonic/gin v1.7.1
+	github.com/spf13/cobra v1.1.3
+	github.com/spf13/viper v1.7.1
+	go.uber.org/dig v1.10.0
+)

backend/go.mod.local

@@ -3,16 +3,18 @@ module crawlab
 go 1.16
 
 replace (
-    github.com/crawlab-team/crawlab-core => /Users/chenjingyang/dev/my-job/gitee/crawlab-core
-	github.com/crawlab-team/goseaweedfs => /Users/chenjingyang/dev/my-job/gitee/goseaweedfs
+    github.com/crawlab-team/crawlab-core => /libs/crawlab-team/crawlab-core
+    github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs
+    github.com/crawlab-team/crawlab-fs => /libs/crawlab-team/crawlab-fs
+    github.com/crawlab-team/crawlab-db => /libs/crawlab-team/crawlab-db
 )
 
 require (
-		github.com/apex/log v1.9.0
-    	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200
-    	github.com/crawlab-team/go-trace v0.1.1
-    	github.com/gin-gonic/gin v1.7.1
-    	github.com/spf13/cobra v1.1.3
-    	github.com/spf13/viper v1.7.1
-    	go.uber.org/dig v1.10.0
+	github.com/apex/log v1.9.0
+    github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200
+    github.com/crawlab-team/go-trace v0.1.1
+    github.com/gin-gonic/gin v1.7.1
+    github.com/spf13/cobra v1.1.3
+    github.com/spf13/viper v1.7.1
+    go.uber.org/dig v1.10.0
 )

backend/go.sum

@@ -63,8 +63,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200 h1:2Tzjkm3VwfvMpeh91DI6gv0wNgtTyVwMfuU2HvX0Uoc=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20220128.1200 h1:aiGdbWNE9awSdoNLAeZF+0TqW7SQqCXoOAZkGtYloXA=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20220128.1200/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY=
 github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
 github.com/crawlab-team/crawlab-db v0.1.3 h1:RqLoXGZEMUH1B8SQB5OcNmJeyY2xILvwyhv4X9faWl4=
 github.com/crawlab-team/crawlab-db v0.1.3/go.mod h1:kPkGZ1P802XdbFFb8byMpZfNG2lWTNoWNRy4beS0/QY=

bin/docker-start-master.sh

@@ -1,4 +1,16 @@
-#!/bin/bash
+# replace base url
+if [ "${CRAWLAB_BASE_URL}" = "" ];
+then
+	:
+else
+	indexpath=/app/dist/index.html
+	sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath}
+	sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath}
+
+	sed -i "s/  <link rel=\"icon\" type=\"image\/x-icon\" href=\"/  <link rel=\"icon\" type=\"image\/x-icon\" href=\"\/${CRAWLAB_BASE_URL}/g"  ${indexpath}
+	sed -i "s/  <link rel=\"stylesheet\" href=\"/  <link rel=\"stylesheet\" href=\"${CRAWLAB_BASE_URL}\//g"  ${indexpath}
+	sed -i "s/  window.VUE_APP_API_BASE_URL = '/  window.VUE_APP_API_BASE_URL = '\/${CRAWLAB_BASE_URL}/g" ${indexpath}
+fi
 
 # start nginx
 service nginx start
@@ -18,3 +30,4 @@ weed server \
 	-volume.port 9999 \
 	-filer \
 	>> /var/log/weed.log 2>&1 &
+

docker-compose.yml

@@ -4,6 +4,7 @@ services:
     image: crawlabteam/crawlab:latest
     container_name: crawlab_master
     environment:
+      CRAWLAB_BASE_URL: crawlab
       CRAWLAB_SERVER_MASTER: Y
       CRAWLAB_MONGO_HOST: mongo
     ports:

frontend/.editorconfig

@@ -0,0 +1,40 @@
+root = true
+
+[*]
+tab_width = 2
+# charset = utf-8
+# end_of_line = lf
+# indent_size = 4
+# indent_style = space
+# insert_final_newline = true
+# max_line_length = 120
+# tab_width = 4
+# trim_trailing_whitespace = true
+
+[*.ts]
+indent_size = 2
+
+[*.scss]
+indent_size = 2
+
+[{*.ats, *.ts}]
+# indent_size = 2
+# tab_width = 2
+
+[{*.js, *.cjs}]
+# indent_size = 2
+# tab_width = 2
+
+[{*.sht, *.html, *.shtm, *.shtml, *.htm, *.ng}]
+# indent_size = 2
+# tab_width = 2
+
+[{.analysis_options, *.yml, *.yaml}]
+# indent_size = 2
+
+[{.babelrc, .prettierrc, .stylelintrc, .eslintrc, jest.config, *.json, *.jsb3, *.jsb2, *.bowerrc}]
+# indent_size = 2
+
+[vue.config.js]
+indent_size = 2
+tab_width = 2

frontend/.env.analyze

@@ -0,0 +1,2 @@
+NODE_ENV='analyze'
+VUE_APP_API_BASE_URL=http://localhost:8000

frontend/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@crawlab/app",
-  "version": "0.6.0-beta.20211226.1800",
-  "description": "",
+  "version": "0.6.0-beta.20220218.1800",
   "scripts": {
     "serve": "vue-cli-service serve",
     "serve:dist": "serve dist",
     "build": "vue-cli-service build",
     "build:development": "vue-cli-service build --mode development",
-    "build:docker": "vue-cli-service build --mode docker"
+    "build:docker": "vue-cli-service build --mode docker",
+    "build:analyze": "vue-cli-service build --mode analyze"
   },
   "author": {
     "name": "Marvin Zhang",
@@ -15,13 +15,31 @@
   },
   "license": "BSD-3-Clause",
   "dependencies": {
-    "crawlab-ui": "0.6.0-beta.20211229.1830",
-    "vue": "3.0.11",
+    "@element-plus/icons": "^0.0.11",
+    "@fortawesome/fontawesome-svg-core": "^1.3.0",
+    "@fortawesome/free-brands-svg-icons": "^6.0.0",
+    "@fortawesome/free-regular-svg-icons": "^6.0.0",
+    "@fortawesome/free-solid-svg-icons": "^6.0.0",
+    "@fortawesome/vue-fontawesome": "^3.0.0-5",
+    "atom-material-icons": "^3.0.0",
+    "codemirror": "^5.59.1",
+    "crawlab-ui": "^0.6.0-beta.20220213.1200",
+    "echarts": "^5.1.2",
+    "element-plus": "^1.3.0-beta.9",
+    "vue": "^3.2",
     "vue-router": "^4.0.11"
   },
   "devDependencies": {
-    "@vue/cli-service": "^4.5.13",
+    "@babel/core": "^7.17.2",
+    "@babel/preset-env": "^7.16.11",
+    "@babel/preset-typescript": "^7.16.7",
+    "@vue/cli-plugin-babel": "^4.5.15",
+    "@vue/cli-service": "^4.5.15",
     "@vue/compiler-sfc": "^3.0.11",
-    "serve": "^13.0.2"
+    "babel-loader": "^8.2.3",
+    "copy-webpack-plugin": "^6.0",
+    "raw-loader": "^4.0.2",
+    "serve": "^13.0.2",
+    "webpack-bundle-analyzer": "^4.5.0"
   }
 }

frontend/src/main.ts

@@ -1,4 +1,5 @@
 import 'crawlab-ui/dist/crawlab-ui.css';
+import 'vue';
 import {createApp} from 'crawlab-ui';
 
 (async function () {

frontend/vue.config.js

@@ -1,3 +1,14 @@
+const path = require("path")
+const CopyWebpackPlugin = require('copy-webpack-plugin')
+const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin
+
+const alias = {
+  'crawlab-ui$': 'crawlab-ui/dist/crawlab-ui.umd.min.js',
+  'element-plus$': 'element-plus/dist/index.full.min.js',
+  'echarts$': 'echarts/dist/echarts.min.js',
+  'codemirror$': 'codemirror/lib/codemirror.js',
+}
+
 const optimization = {
   splitChunks: {
     chunks: 'initial',
@@ -6,17 +17,17 @@ const optimization = {
     maxAsyncRequests: 3,
     cacheGroups: {
       defaultVendors: {
-        test: /[\\/]node_modules[\\/]]/,
+        test: /[\\/]node_modules[\\/]/,
         priority: -10,
-        reuseExistingChunk: true
+        reuseExistingChunk: true,
       },
       default: {
         minChunks: 2,
         priority: -20,
-        reuseExistingChunk: true
-      }
-    }
-  }
+        reuseExistingChunk: true,
+      },
+    },
+  },
 }
 
 const config = {
@@ -31,8 +42,27 @@ const config = {
   outputDir: './dist',
   configureWebpack: {
     optimization,
+    resolve: {
+      alias,
+    },
     plugins: []
   }
 }
 
+if (['development', 'local'].includes(process.env.NODE_ENV)) {
+  // do nothing
+} else if (['production', 'docker'].includes(process.env.NODE_ENV)) {
+  config.configureWebpack.plugins.push(new CopyWebpackPlugin({
+    patterns: [
+      {
+        from: path.resolve(__dirname, 'public/js'),
+      }
+    ]
+  }))
+} else if (['analyze'].includes(process.env.NODE_ENV)) {
+  config.configureWebpack.plugins.push(new BundleAnalyzerPlugin({
+    analyzePort: 8890,
+  }))
+}
+
 module.exports = config

scripts/validate-backend.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+IFS=$'\n'
+pattern=^replace
+content=$(cat ./backend/go.mod)
+for line in $content
+do
+	if [[ $line =~ $pattern ]]; then
+		echo "Invalid ./backend/go.mod, which should not contain \"^replace\""
+		exit 1
+	fi
+done

workspace/.gitignore

@@ -0,0 +1 @@
+data/

workspace/localdev/README.md

@@ -0,0 +1,17 @@
+# Local Dev
+
+For those who want to develop or contribute to crawlab project can use `docker-compose.yml` to start necessary containers (fs, mongo) to support local development.
+
+Please make sure you have installed [Docker](https://docker.io) and [Docker-Compose](https://docs.docker.com/compose/).
+
+## Start Containers
+
+```bash
+docker-compose up -d
+```
+
+## Stop Containers
+
+```bash
+docker-compose down
+```

workspace/localdev/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '2'
+
+services:
+
+  fs:
+    image: chrislusf/seaweedfs:2.79 # use a remote image
+    container_name: crawlab_localdev_fs
+    restart: always
+#    command: "server -dir /data -master.dir /data -volume.dir.idx /data -ip localhost -ip.bind 0.0.0.0 -filer -encryptVolumeData" # auth
+    command: "server -dir /data -master.dir /data -volume.dir.idx /data -ip localhost -ip.bind 0.0.0.0 -filer" # no auth
+    ports:
+      - 8888:8888
+    volumes:
+      - ./data/fs:/data
+  
+  mongo:
+    image: mongo:5
+    container_name: crawlab_localdev_mongo
+    restart: always
+    ports:
+      - 27017:27017
+    volumes:
+      - ./data/mongo:/data