From 82be30999d0d9c8a4dee060fce6f144e39a59f9c Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 2 Aug 2021 13:53:35 +0800 Subject: [PATCH] fixed issues with task mode --- backend/go.mod | 2 +- backend/go.sum | 9 ++----- docker_init.sh | 31 +++++++++++++++++------ docs/config.md | 17 +++++++++++++ frontend/src/components/tag/tag.ts | 4 +-- frontend/src/components/task/TaskMode.vue | 10 ++++---- frontend/src/constants/task.ts | 2 +- frontend/src/interfaces/views/task.d.ts | 4 +-- frontend/src/utils/task.ts | 4 +-- 9 files changed, 55 insertions(+), 28 deletions(-) create mode 100644 docs/config.md diff --git a/backend/go.mod b/backend/go.mod index e821ef12..b70e650e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,7 +4,7 @@ go 1.15 require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20210729.1450 + github.com/crawlab-team/crawlab-core v0.6.0-beta.20210802.1344 github.com/crawlab-team/go-trace v0.1.0 github.com/gin-gonic/gin v1.6.3 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.sum b/backend/go.sum index 80cecf37..cfa705ff 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -28,7 +28,6 @@ github.com/StackExchange/wmi v1.2.0 h1:noJEYkMQVlFCEAc+2ma5YyRhlfjcWfZqk5sBRYozd github.com/StackExchange/wmi v1.2.0/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw= github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= -github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= @@ -75,8 +74,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crawlab-team/crawlab-core v0.0.1 h1:tZ9Rlji5L5uoZ6nRZFqnSGmEl5TdmdxAyxueJDKfUl8= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20210729.1450 h1:6YbX6k9Kd4OwJ3Nod8UZ9DkQCzK+Qy0DAJC+2u21dZM= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20210729.1450/go.mod h1:vDgvJZFtwUNWmeaZ8Pckk9yICC8WnVogmHBEaaHeDM8= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20210802.1344 h1:iRFkt5TAQM4T6NWeJfcM/yT1USqskv5SmqKYiDfHb9I= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20210802.1344/go.mod h1:3K3ZIbZOmks7nJT/nfR338SrXFPHzD0AT1F7TSVuI/w= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.1.1 h1:156h2fbbFKXAHs1mxprqRFC8zs2nrdyaG9JKG7patVw= github.com/crawlab-team/crawlab-db v0.1.1/go.mod h1:t0VidSjXKzQgACqNSQV5wusXncFtL6lGEiQTbLfNR04= @@ -119,7 +118,6 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/gamexg/proxyclient v0.0.0-20210207161252-499908056324/go.mod h1:Y1lsrg1lNY/7qdJ6masxXhIOUHajrlcixeMMWU59cjg= github.com/gavv/httpexpect/v2 v2.2.0/go.mod h1:lnd0TqJLrP+wkJk3SFwtrpSlOAZQ7HaaIFuOYbgqgUM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= @@ -421,8 +419,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= -github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -440,7 +436,6 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shadowsocks/shadowsocks-go v0.0.0-20200409064450-3e585ff90601/go.mod h1:mttDPaeLm87u74HMrP+n2tugXvIKWcwff/cqSX0lehY= github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty4DZO54I4FGqIpto= github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= diff --git a/docker_init.sh b/docker_init.sh index e46ab7bf..ad3d9815 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -23,12 +23,6 @@ sed -i "s?VUE_APP_API_BASE_URL?\/api?g" ${jspath} # sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath} # fi -# start nginx -service nginx start - -# start seaweedfs server -weed server -dir /data -master.dir /data -volume.dir.idx /data -ip localhost -ip.bind 0.0.0.0 -filer >> /var/log/weed.log 2>&1 & - #grant script # chmod +x /app/backend/scripts/*.sh @@ -51,8 +45,29 @@ Host * EOF # start backend -if [ "${CRAWLAB_SERVER_MASTER}" = "Y" ]; +if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then + # start nginx + service nginx start + + # start seaweedfs server + seaweedfsDataPath=/data/seaweedfs + if [ -e ${seaweedfsDataPath} ]; then + : + else + mkdir -p ${seaweedfsDataPath} + fi + weed server \ + -dir /data \ + -master.dir ${seaweedfsDataPath} \ + -volume.dir.idx ${seaweedfsDataPath} \ + -ip localhost \ + -volume.port 9999 \ + -filer \ + >> /var/log/weed.log 2>&1 & + + # start crawlab crawlab-server master -then +else + # start crawlab crawlab-server worker fi diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 00000000..9ba6e056 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,17 @@ +# Config + +#### Environment Variable List + +Name | Description | Example | Default +---|--- | --- | --- +CRAWLAB_GRPC_ADDRESS| Target gRPC address the nodes are connecting to | 192.168.0.1:9666 | localhost:9666 +CRAWLAB_GRPC_SERVER_ADDRESS| Address that the gRPC server is listening to (master node only) | 0.0.0.0:9666 | 0.0.0.0:9666 +CRAWLAB_GRPC_AUTHKEY| The token that gRPC clients and server use for authentication | youcanneverguess | Crawlab2021! +CRAWLAB_NODE_MASTER | Whether the current node is a master or worker node (Y: master; N: worker) | Y | Y +CRAWLAB_SERVER_HOST | IP host that the API listens to (master node only) | 0.0.0.0 | 0.0.0.0 +CRAWLAB_SERVER_PORT | IP port that the API listens to (master node only) | 8000 | 8000 +CRAWLAB_TASK_HANDLER_MAXRUNNERS | Max number of task runners (concurrent spider tasks) that a node can run | 16 | 8 +CRAWLAB_FS_FILER_PROXY |Filer API endpoint that Crawlab's filer proxy links to |http://filer-server:8888 | http://localhost:8888 +CRAWLAB_FS_FILER_URL |Crawlab's Filer API endpoint |http://crawlab-web-api:8000/filer | http://localhost:8000/filer +CRAWLAB_FS_FILER_AUTHKEY |Crawlab's Filer API auth key token | youcanneverguess | Crawlab2021! + diff --git a/frontend/src/components/tag/tag.ts b/frontend/src/components/tag/tag.ts index db0afe30..c2976db3 100644 --- a/frontend/src/components/tag/tag.ts +++ b/frontend/src/components/tag/tag.ts @@ -1,7 +1,7 @@ import {useRoute} from 'vue-router'; import {computed} from 'vue'; import { - TASK_MODE_ALL, + TASK_MODE_ALL_NODES, TASK_MODE_RANDOM, TASK_MODE_SELECTED_NODE_TAGS, TASK_MODE_SELECTED_NODES @@ -31,7 +31,7 @@ const useTag = (store: Store) => { // options for default mode const modeOptions: SelectOption[] = [ {value: TASK_MODE_RANDOM, label: 'Random Node'}, - {value: TASK_MODE_ALL, label: 'All Nodes'}, + {value: TASK_MODE_ALL_NODES, label: 'All Nodes'}, {value: TASK_MODE_SELECTED_NODES, label: 'Selected Nodes'}, {value: TASK_MODE_SELECTED_NODE_TAGS, label: 'Selected Tags'}, ]; diff --git a/frontend/src/components/task/TaskMode.vue b/frontend/src/components/task/TaskMode.vue index 5cf40990..8750c184 100644 --- a/frontend/src/components/task/TaskMode.vue +++ b/frontend/src/components/task/TaskMode.vue @@ -10,7 +10,7 @@