git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

fixed issues with task mode

Browse Source
This commit is contained in:
marvzhang
2021-08-02 13:53:35 +08:00
parent 87f13483e6
commit 57569862a5
9 changed files with 55 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/go.mod
View File

@@ -4,7 +4,7 @@ go 1.15
require (
github.com/apex/log v1.9.0
github.com/crawlab-team/crawlab-core v0.6.0-beta.20210729.1450
github.com/crawlab-team/crawlab-core v0.6.0-beta.20210802.1344
github.com/crawlab-team/go-trace v0.1.0
github.com/gin-gonic/gin v1.6.3
github.com/spf13/cobra v1.1.3

9
backend/go.sum
View File

@@ -28,7 +28,6 @@ github.com/StackExchange/wmi v1.2.0 h1:noJEYkMQVlFCEAc+2ma5YyRhlfjcWfZqk5sBRYozd
github.com/StackExchange/wmi v1.2.0/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw=
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
@@ -75,8 +74,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/crawlab-team/crawlab-core v0.0.1 h1:tZ9Rlji5L5uoZ6nRZFqnSGmEl5TdmdxAyxueJDKfUl8=
github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
github.com/crawlab-team/crawlab-core v0.6.0-beta.20210729.1450 h1:6YbX6k9Kd4OwJ3Nod8UZ9DkQCzK+Qy0DAJC+2u21dZM=
github.com/crawlab-team/crawlab-core v0.6.0-beta.20210729.1450/go.mod h1:vDgvJZFtwUNWmeaZ8Pckk9yICC8WnVogmHBEaaHeDM8=
github.com/crawlab-team/crawlab-core v0.6.0-beta.20210802.1344 h1:iRFkt5TAQM4T6NWeJfcM/yT1USqskv5SmqKYiDfHb9I=
github.com/crawlab-team/crawlab-core v0.6.0-beta.20210802.1344/go.mod h1:3K3ZIbZOmks7nJT/nfR338SrXFPHzD0AT1F7TSVuI/w=
github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
github.com/crawlab-team/crawlab-db v0.1.1 h1:156h2fbbFKXAHs1mxprqRFC8zs2nrdyaG9JKG7patVw=
github.com/crawlab-team/crawlab-db v0.1.1/go.mod h1:t0VidSjXKzQgACqNSQV5wusXncFtL6lGEiQTbLfNR04=
@@ -119,7 +118,6 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gamexg/proxyclient v0.0.0-20210207161252-499908056324/go.mod h1:Y1lsrg1lNY/7qdJ6masxXhIOUHajrlcixeMMWU59cjg=
github.com/gavv/httpexpect/v2 v2.2.0/go.mod h1:lnd0TqJLrP+wkJk3SFwtrpSlOAZQ7HaaIFuOYbgqgUM=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w=
@@ -421,8 +419,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -440,7 +436,6 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shadowsocks/shadowsocks-go v0.0.0-20200409064450-3e585ff90601/go.mod h1:mttDPaeLm87u74HMrP+n2tugXvIKWcwff/cqSX0lehY=
github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty4DZO54I4FGqIpto=
github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=

31
docker_init.sh
View File

@@ -23,12 +23,6 @@ sed -i "s?VUE_APP_API_BASE_URL?\/api?g" ${jspath}
# sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath}
# fi
# start nginx
service nginx start
# start seaweedfs server
weed server -dir /data -master.dir /data -volume.dir.idx /data -ip localhost -ip.bind 0.0.0.0 -filer >> /var/log/weed.log 2>&1 &
#grant script
# chmod +x /app/backend/scripts/*.sh
@@ -51,8 +45,29 @@ Host *
EOF
# start backend
if [ "${CRAWLAB_SERVER_MASTER}" = "Y" ];
if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then
# start nginx
service nginx start
# start seaweedfs server
seaweedfsDataPath=/data/seaweedfs
if [ -e ${seaweedfsDataPath} ]; then
:
else
mkdir -p ${seaweedfsDataPath}
fi
weed server \
-dir /data \
-master.dir ${seaweedfsDataPath} \
-volume.dir.idx ${seaweedfsDataPath} \
-ip localhost \
-volume.port 9999 \
-filer \
>> /var/log/weed.log 2>&1 &
# start crawlab
crawlab-server master
then
else
# start crawlab
crawlab-server worker
fi

17
docs/config.md Normal file
View File

@@ -0,0 +1,17 @@
# Config
#### Environment Variable List
Name | Description | Example | Default
---|--- | --- | ---
CRAWLAB_GRPC_ADDRESS| Target gRPC address the nodes are connecting to | 192.168.0.1:9666 | localhost:9666
CRAWLAB_GRPC_SERVER_ADDRESS| Address that the gRPC server is listening to (master node only) | 0.0.0.0:9666 | 0.0.0.0:9666
CRAWLAB_GRPC_AUTHKEY| The token that gRPC clients and server use for authentication | youcanneverguess | Crawlab2021!
CRAWLAB_NODE_MASTER | Whether the current node is a master or worker node (Y: master; N: worker) | Y | Y
CRAWLAB_SERVER_HOST | IP host that the API listens to (master node only) | 0.0.0.0 | 0.0.0.0
CRAWLAB_SERVER_PORT | IP port that the API listens to (master node only) | 8000 | 8000
CRAWLAB_TASK_HANDLER_MAXRUNNERS | Max number of task runners (concurrent spider tasks) that a node can run | 16 | 8
CRAWLAB_FS_FILER_PROXY |Filer API endpoint that Crawlab's filer proxy links to |http://filer-server:8888 | http://localhost:8888
CRAWLAB_FS_FILER_URL |Crawlab's Filer API endpoint |http://crawlab-web-api:8000/filer | http://localhost:8000/filer
CRAWLAB_FS_FILER_AUTHKEY |Crawlab's Filer API auth key token | youcanneverguess | Crawlab2021!

4
frontend/src/components/tag/tag.ts
View File

@@ -1,7 +1,7 @@
import {useRoute} from 'vue-router';
import {computed} from 'vue';
import {
TASK_MODE_ALL,
TASK_MODE_ALL_NODES,
TASK_MODE_RANDOM,
TASK_MODE_SELECTED_NODE_TAGS,
TASK_MODE_SELECTED_NODES
@@ -31,7 +31,7 @@ const useTag = (store: Store<RootStoreState>) => {
// options for default mode
const modeOptions: SelectOption[] = [
{value: TASK_MODE_RANDOM, label: 'Random Node'},
{value: TASK_MODE_ALL, label: 'All Nodes'},
{value: TASK_MODE_ALL_NODES, label: 'All Nodes'},
{value: TASK_MODE_SELECTED_NODES, label: 'Selected Nodes'},
{value: TASK_MODE_SELECTED_NODE_TAGS, label: 'Selected Tags'},
];

10
frontend/src/components/task/TaskMode.vue
View File

@@ -10,7 +10,7 @@
<script lang="ts">
import {computed, defineComponent} from 'vue';
import {
TASK_MODE_ALL,
TASK_MODE_ALL_NODES,
TASK_MODE_RANDOM,
TASK_MODE_SELECTED_NODE_TAGS,
TASK_MODE_SELECTED_NODES
@@ -30,7 +30,7 @@ export default defineComponent({
switch (mode) {
case TASK_MODE_RANDOM:
return 'warning';
case TASK_MODE_ALL:
case TASK_MODE_ALL_NODES:
return 'success';
case TASK_MODE_SELECTED_NODES:
return 'primary';
@@ -46,7 +46,7 @@ export default defineComponent({
switch (mode) {
case TASK_MODE_RANDOM:
return 'Random';
case TASK_MODE_ALL:
case TASK_MODE_ALL_NODES:
return 'All Nodes';
case TASK_MODE_SELECTED_NODES:
return 'Nodes';
@@ -62,7 +62,7 @@ export default defineComponent({
switch (mode) {
case TASK_MODE_RANDOM:
return ['fa', 'random'];
case TASK_MODE_ALL:
case TASK_MODE_ALL_NODES:
return ['fa', 'sitemap'];
case TASK_MODE_SELECTED_NODES:
return ['fa', 'network-wired'];
@@ -78,7 +78,7 @@ export default defineComponent({
switch (mode) {
case TASK_MODE_RANDOM:
return 'Run on a random node';
case TASK_MODE_ALL:
case TASK_MODE_ALL_NODES:
return 'Run on all nodes';
case TASK_MODE_SELECTED_NODES:
return 'Run on selected nodes';

2
frontend/src/constants/task.ts
View File

@@ -6,6 +6,6 @@ export const TASK_STATUS_CANCELLED = 'cancelled';
export const TASK_STATUS_ABNORMAL = 'abnormal';
export const TASK_MODE_RANDOM = 'random';
export const TASK_MODE_ALL = 'all';
export const TASK_MODE_ALL_NODES = 'all-nodes';
export const TASK_MODE_SELECTED_NODES = 'selected-nodes';
export const TASK_MODE_SELECTED_NODE_TAGS = 'selected-node-tags';

4
frontend/src/interfaces/views/task.d.ts vendored
View File

@@ -1,5 +1,5 @@
import {
TASK_MODE_ALL,
TASK_MODE_ALL_NODES,
TASK_MODE_RANDOM,
TASK_MODE_SELECTED_NODE_TAGS,
TASK_MODE_SELECTED_NODES,
@@ -55,7 +55,7 @@ declare global {
type TaskMode =
TASK_MODE_RANDOM
| TASK_MODE_ALL
| TASK_MODE_ALL_NODES
| TASK_MODE_SELECTED_NODES
| TASK_MODE_SELECTED_NODE_TAGS;
}

4
frontend/src/utils/task.ts
View File

@@ -1,5 +1,5 @@
import {
TASK_MODE_ALL,
TASK_MODE_ALL_NODES,
TASK_MODE_RANDOM,
TASK_MODE_SELECTED_NODE_TAGS,
TASK_MODE_SELECTED_NODES,
@@ -34,7 +34,7 @@ export const isCancellable = (status: TaskStatus): boolean => {
export const getModeOptions = (): SelectOption[] => {
return [
{value: TASK_MODE_RANDOM, label: 'Random Node'},
{value: TASK_MODE_ALL, label: 'All Nodes'},
{value: TASK_MODE_ALL_NODES, label: 'All Nodes'},
{value: TASK_MODE_SELECTED_NODES, label: 'Selected Nodes'},
{value: TASK_MODE_SELECTED_NODE_TAGS, label: 'Selected Tags'},
];