From a34017cac12c71895f77363490e02d3da9ab9d1b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 12:00:53 +0800 Subject: [PATCH 01/12] updated devops files --- .github/workflows/dockerpush.yml | 4 +++ devops/develop/crawlab-master.yaml | 49 ++++++++++++++++++++++++++++++ devops/develop/crawlab-worker.yaml | 32 +++++++++++++++++++ devops/develop/mongo-pv.yaml | 28 +++++++++++++++++ devops/develop/mongo.yaml | 41 +++++++++++++++++++++++++ devops/develop/ns.yaml | 4 +++ devops/develop/redis.yaml | 34 +++++++++++++++++++++ devops/release/crawlab-master.yaml | 4 +++ devops/release/crawlab-worker.yaml | 6 +++- k8s/crawlab-master.yaml | 4 +++ k8s/crawlab-worker.yaml | 4 +++ 11 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 devops/develop/crawlab-master.yaml create mode 100644 devops/develop/crawlab-worker.yaml create mode 100644 devops/develop/mongo-pv.yaml create mode 100644 devops/develop/mongo.yaml create mode 100644 devops/develop/ns.yaml create mode 100644 devops/develop/redis.yaml diff --git a/.github/workflows/dockerpush.yml b/.github/workflows/dockerpush.yml index 61be3f8a..be968fe2 100644 --- a/.github/workflows/dockerpush.yml +++ b/.github/workflows/dockerpush.yml @@ -6,6 +6,7 @@ on: branches: - master - release + - develop # Publish `v1.2.3` tags as releases. tags: @@ -60,3 +61,6 @@ jobs: if [ "$VERSION" == "release" ]; then curl ${{ secrets.JENKINS_RELEASE_URL }} fi + if [ "$VERSION" == "develop" ]; then + curl ${{ secrets.JENKINS_DEVELOP_URL }} + fi diff --git a/devops/develop/crawlab-master.yaml b/devops/develop/crawlab-master.yaml new file mode 100644 index 00000000..68015e30 --- /dev/null +++ b/devops/develop/crawlab-master.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: crawlab + namespace: crawlab-develop +spec: + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30108 + selector: + app: crawlab-master + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-master + namespace: crawlab-develop +spec: + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-master + template: + metadata: + labels: + app: crawlab-master + spec: + containers: + - image: tikazyq/crawlab:develop + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "Y" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + - name: CRAWLAB_SETTING_ALLOWREGISTER + value: "Y" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" + ports: + - containerPort: 8080 + name: crawlab \ No newline at end of file diff --git a/devops/develop/crawlab-worker.yaml b/devops/develop/crawlab-worker.yaml new file mode 100644 index 00000000..7522a581 --- /dev/null +++ b/devops/develop/crawlab-worker.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-worker + namespace: crawlab-develop +spec: + replicas: 2 + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-worker + template: + metadata: + labels: + app: crawlab-worker + spec: + containers: + - image: tikazyq/crawlab:develop + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "N" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" + \ No newline at end of file diff --git a/devops/develop/mongo-pv.yaml b/devops/develop/mongo-pv.yaml new file mode 100644 index 00000000..e47a0ed9 --- /dev/null +++ b/devops/develop/mongo-pv.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: mongo-pv-volume-develop + namespace: crawlab-develop + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data/crawlab-develop/mongodb/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-pv-claim-develop + namespace: crawlab-develop +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi \ No newline at end of file diff --git a/devops/develop/mongo.yaml b/devops/develop/mongo.yaml new file mode 100644 index 00000000..21a685bb --- /dev/null +++ b/devops/develop/mongo.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongo + namespace: crawlab-develop +spec: + ports: + - port: 27017 + selector: + app: mongo + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: mongo + namespace: crawlab-develop +spec: + selector: + matchLabels: + app: mongo + strategy: + type: Recreate + template: + metadata: + labels: + app: mongo + spec: + containers: + - image: mongo:4 + name: mongo + ports: + - containerPort: 27017 + name: mongo + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + volumes: + - name: mongo-persistent-storage + persistentVolumeClaim: + claimName: mongo-pv-claim-develop \ No newline at end of file diff --git a/devops/develop/ns.yaml b/devops/develop/ns.yaml new file mode 100644 index 00000000..bef4391e --- /dev/null +++ b/devops/develop/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: crawlab-develop \ No newline at end of file diff --git a/devops/develop/redis.yaml b/devops/develop/redis.yaml new file mode 100644 index 00000000..204e3242 --- /dev/null +++ b/devops/develop/redis.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: crawlab-develop +spec: + ports: + - port: 6379 + selector: + app: redis + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: redis + namespace: crawlab-develop +spec: + selector: + matchLabels: + app: redis + strategy: + type: Recreate + template: + metadata: + labels: + app: redis + spec: + containers: + - image: redis + name: redis + ports: + - containerPort: 6379 + name: redis \ No newline at end of file diff --git a/devops/release/crawlab-master.yaml b/devops/release/crawlab-master.yaml index e886fdd3..759ef6b4 100644 --- a/devops/release/crawlab-master.yaml +++ b/devops/release/crawlab-master.yaml @@ -40,6 +40,10 @@ spec: value: "redis" - name: CRAWLAB_SETTING_ALLOWREGISTER value: "Y" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" ports: - containerPort: 8080 name: crawlab \ No newline at end of file diff --git a/devops/release/crawlab-worker.yaml b/devops/release/crawlab-worker.yaml index 5bc2f396..986ee0c0 100644 --- a/devops/release/crawlab-worker.yaml +++ b/devops/release/crawlab-worker.yaml @@ -24,4 +24,8 @@ spec: - name: CRAWLAB_MONGO_HOST value: "mongo" - name: CRAWLAB_REDIS_ADDRESS - value: "redis" \ No newline at end of file + value: "redis" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" diff --git a/k8s/crawlab-master.yaml b/k8s/crawlab-master.yaml index b87f6881..62ac6bca 100644 --- a/k8s/crawlab-master.yaml +++ b/k8s/crawlab-master.yaml @@ -38,6 +38,10 @@ spec: value: "mongo" - name: CRAWLAB_REDIS_ADDRESS value: "redis" + # - name: CRAWLAB_SERVER_LANG_NODE + # value: "Y" + # - name: CRAWLAB_SERVER_LANG_JAVA + # value: "Y" ports: - containerPort: 8080 name: crawlab diff --git a/k8s/crawlab-worker.yaml b/k8s/crawlab-worker.yaml index b54a7796..35ae200a 100644 --- a/k8s/crawlab-worker.yaml +++ b/k8s/crawlab-worker.yaml @@ -25,3 +25,7 @@ spec: value: "mongo" - name: CRAWLAB_REDIS_ADDRESS value: "redis" + # - name: CRAWLAB_SERVER_LANG_NODE + # value: "Y" + # - name: CRAWLAB_SERVER_LANG_JAVA + # value: "Y" From 42a2eac3f5e622c83475edbc8be261ad0ff50045 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 12:23:03 +0800 Subject: [PATCH 02/12] updated install-java.sh --- backend/scripts/install-java.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh index e98de224..d01d0247 100644 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -4,6 +4,7 @@ touch /tmp/install-java.lock # install java +apt-get update apt-get install -y default-jdk ln -s /usr/bin/java /usr/local/bin/java From eafb2c70c8770ea84c6ae40b6a63cae949727da2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 14:29:42 +0800 Subject: [PATCH 03/12] code cleanup --- Dockerfile | 4 +++- Dockerfile.local | 4 +++- crawlab.conf | 5 ----- {frontend/conf => nginx}/crawlab.conf | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 crawlab.conf rename {frontend/conf => nginx}/crawlab.conf (94%) diff --git a/Dockerfile b/Dockerfile index d161ef1e..48f41d5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,9 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server # copy frontend files COPY --from=frontend-build /app/dist /app/dist -COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d + +# copy nginx config files +COPY ./nginx/crawlab.conf /etc/nginx/conf.d # working directory WORKDIR /app/backend diff --git a/Dockerfile.local b/Dockerfile.local index eb06b7e6..d49df7db 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -49,7 +49,9 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server # copy frontend files COPY --from=frontend-build /app/dist /app/dist -COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d + +# copy nginx config files +COPY ./nginx/crawlab.conf /etc/nginx/conf.d # working directory WORKDIR /app/backend diff --git a/crawlab.conf b/crawlab.conf deleted file mode 100644 index f0b7cef2..00000000 --- a/crawlab.conf +++ /dev/null @@ -1,5 +0,0 @@ -server { - listen 8080; - root /opt/crawlab/frontend/dist; - index index.html; -} \ No newline at end of file diff --git a/frontend/conf/crawlab.conf b/nginx/crawlab.conf similarity index 94% rename from frontend/conf/crawlab.conf rename to nginx/crawlab.conf index 80282fc4..fe911819 100644 --- a/frontend/conf/crawlab.conf +++ b/nginx/crawlab.conf @@ -2,7 +2,6 @@ server { gzip on; gzip_min_length 1k; gzip_buffers 4 16k; - #gzip_http_version 1.0; gzip_comp_level 2; gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; gzip_vary off; From e6826b4de165925ee73f3bbedf1e36b24a254162 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 14:33:41 +0800 Subject: [PATCH 04/12] fixed https://github.com/crawlab-team/crawlab/issues/616 --- nginx/crawlab.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/nginx/crawlab.conf b/nginx/crawlab.conf index fe911819..36c9f2b1 100644 --- a/nginx/crawlab.conf +++ b/nginx/crawlab.conf @@ -13,5 +13,6 @@ server { location /api/ { rewrite /api/(.*) /$1 break; proxy_pass http://localhost:8000/; + client_max_body_size 50m; } } From cef919a2dec89315fb540e0cc0acc4f781e3c9a2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 14:48:03 +0800 Subject: [PATCH 05/12] updated imagePullPolicy --- devops/develop/crawlab-master.yaml | 1 + devops/develop/crawlab-worker.yaml | 1 + devops/release/crawlab-master.yaml | 1 + devops/release/crawlab-worker.yaml | 1 + k8s/crawlab-master.yaml | 1 + k8s/crawlab-worker.yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/devops/develop/crawlab-master.yaml b/devops/develop/crawlab-master.yaml index 68015e30..1168c404 100644 --- a/devops/develop/crawlab-master.yaml +++ b/devops/develop/crawlab-master.yaml @@ -30,6 +30,7 @@ spec: spec: containers: - image: tikazyq/crawlab:develop + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/devops/develop/crawlab-worker.yaml b/devops/develop/crawlab-worker.yaml index 7522a581..096f0ccc 100644 --- a/devops/develop/crawlab-worker.yaml +++ b/devops/develop/crawlab-worker.yaml @@ -17,6 +17,7 @@ spec: spec: containers: - image: tikazyq/crawlab:develop + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/devops/release/crawlab-master.yaml b/devops/release/crawlab-master.yaml index 759ef6b4..342d1300 100644 --- a/devops/release/crawlab-master.yaml +++ b/devops/release/crawlab-master.yaml @@ -30,6 +30,7 @@ spec: spec: containers: - image: tikazyq/crawlab:release + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/devops/release/crawlab-worker.yaml b/devops/release/crawlab-worker.yaml index 986ee0c0..bf96607e 100644 --- a/devops/release/crawlab-worker.yaml +++ b/devops/release/crawlab-worker.yaml @@ -17,6 +17,7 @@ spec: spec: containers: - image: tikazyq/crawlab:release + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/k8s/crawlab-master.yaml b/k8s/crawlab-master.yaml index 62ac6bca..758cc051 100644 --- a/k8s/crawlab-master.yaml +++ b/k8s/crawlab-master.yaml @@ -30,6 +30,7 @@ spec: spec: containers: - image: tikazyq/crawlab:latest + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/k8s/crawlab-worker.yaml b/k8s/crawlab-worker.yaml index 35ae200a..2efcd556 100644 --- a/k8s/crawlab-worker.yaml +++ b/k8s/crawlab-worker.yaml @@ -17,6 +17,7 @@ spec: spec: containers: - image: tikazyq/crawlab:latest + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER From fadc62fcc86028284a0491ffbf6bd2328002a3c2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 15:09:15 +0800 Subject: [PATCH 06/12] updated crawlab.conf --- nginx/crawlab.conf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nginx/crawlab.conf b/nginx/crawlab.conf index 36c9f2b1..d04da86c 100644 --- a/nginx/crawlab.conf +++ b/nginx/crawlab.conf @@ -6,13 +6,13 @@ server { gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; gzip_vary off; gzip_disable "MSIE [1-6]\."; - listen 8080; - root /app/dist; - index index.html; + client_max_body_size 50m; + listen 8080; + root /app/dist; + index index.html; - location /api/ { - rewrite /api/(.*) /$1 break; - proxy_pass http://localhost:8000/; - client_max_body_size 50m; - } + location /api/ { + rewrite /api/(.*) /$1 break; + proxy_pass http://localhost:8000/; + } } From e24f4fced7b1c32581b9e3d52b16c8df1194315b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 07:48:48 +0800 Subject: [PATCH 07/12] updated logs --- docker_init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker_init.sh b/docker_init.sh index 9c51fa83..19cf8471 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -26,14 +26,14 @@ service nginx start if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; then echo "installing node.js" - /bin/sh /app/backend/scripts/install-nodejs.sh & + /bin/sh /app/backend/scripts/install-nodejs.sh >> /var/log/install-nodejs.sh.log 2>&1 & fi # install languages: Java if [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ]; then echo "installing java" - /bin/sh /app/backend/scripts/install-java.sh & + /bin/sh /app/backend/scripts/install-java.sh >> /var/log/install-java.sh.log 2>&1 & fi # generate ssh From 9e24d22ee1d876ccf2ad66c558e9cb9e620ace46 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 08:09:12 +0800 Subject: [PATCH 08/12] updated install-java.sh --- backend/constants/register.go | 7 +++++++ backend/model/node.go | 1 + backend/scripts/install-java.sh | 3 +-- backend/services/node.go | 8 ++++++++ backend/services/register/register.go | 6 ++++-- 5 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 backend/constants/register.go mode change 100644 => 100755 backend/scripts/install-java.sh diff --git a/backend/constants/register.go b/backend/constants/register.go new file mode 100644 index 00000000..ad38e7a3 --- /dev/null +++ b/backend/constants/register.go @@ -0,0 +1,7 @@ +package constants + +const ( + RegisterTypeMac = "mac" + RegisterTypeIp = "ip" + RegisterTypeHostname = "hostname" +) diff --git a/backend/model/node.go b/backend/model/node.go index 88c4ed66..51cd5d7e 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -20,6 +20,7 @@ type Node struct { Ip string `json:"ip" bson:"ip"` Port string `json:"port" bson:"port"` Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` Description string `json:"description" bson:"description"` // 用于唯一标识节点，可能是mac地址，可能是ip地址 Key string `json:"key" bson:"key"` diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh old mode 100644 new mode 100755 index d01d0247..c42a2901 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -4,8 +4,7 @@ touch /tmp/install-java.lock # install java -apt-get update -apt-get install -y default-jdk +apt-get update && apt-get install -y default-jdk ln -s /usr/bin/java /usr/local/bin/java # unlock diff --git a/backend/services/node.go b/backend/services/node.go index d6124205..f189ddb3 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -161,6 +161,14 @@ func UpdateNodeData() { log.Errorf(err.Error()) return } + + // 获取Hostname + hostname, err := register.GetRegister().GetHostname() + if err != nil { + log.Errorf(err.Error()) + return + } + // 获取redis的key key, err := register.GetRegister().GetKey() if err != nil { diff --git a/backend/services/register/register.go b/backend/services/register/register.go index ed4e1891..bbdec2f3 100644 --- a/backend/services/register/register.go +++ b/backend/services/register/register.go @@ -1,6 +1,7 @@ package register import ( + "crawlab/constants" "github.com/apex/log" "github.com/spf13/viper" "net" @@ -108,9 +109,9 @@ func GetRegister() Register { } registerType := viper.GetString("server.register.type") - if registerType == "mac" { + if registerType == constants.RegisterTypeMac { register = &MacRegister{} - } else { + } else if registerType == constants.RegisterTypeIp { ip := viper.GetString("server.register.ip") if ip == "" { log.Error("server.register.ip is empty") @@ -120,6 +121,7 @@ func GetRegister() Register { register = &IpRegister{ Ip: ip, } + } else if registerType == constants.RegisterTypeHostname { } log.Info("register type is :" + reflect.TypeOf(register).String()) From a3326da0f540376cc4208bc9e2e9cb8c1f56d484 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 08:56:01 +0800 Subject: [PATCH 09/12] =?UTF-8?q?=E5=8A=A0=E5=85=A5hostname=E6=B3=A8?= =?UTF-8?q?=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 2 +- backend/model/node.go | 20 +++++++--- backend/services/node.go | 3 +- backend/services/register/register.go | 54 +++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 32fce297..1e98a2eb 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -23,7 +23,7 @@ server: master: "Y" secret: "crawlab" register: - # mac地址 或者 ip地址，如果是ip，则需要手动指定IP + # mac地址/ip地址/hostname, 如果是ip，则需要手动指定IP type: "mac" ip: "" lang: # 安装语言环境, Y 为安装，N 为不安装，只对 Docker 有效 diff --git a/backend/model/node.go b/backend/model/node.go index 51cd5d7e..6e29a073 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -43,6 +43,7 @@ func IsMaster() bool { } // 获取本机节点 +// TODO: 这里职责不单一，需要重构 func GetCurrentNode() (Node, error) { // 获得注册的key值 key, err := register.GetRegister().GetKey() @@ -68,7 +69,7 @@ func GetCurrentNode() (Node, error) { //只在master节点运行的时候才检测master节点的信息是否存在 if IsMaster() && err == mgo.ErrNotFound { // 获取本机信息 - ip, mac, key, err := GetNodeBaseInfo() + ip, mac, hostname, key, err := GetNodeBaseInfo() if err != nil { debug.PrintStack() return node, err @@ -81,6 +82,7 @@ func GetCurrentNode() (Node, error) { Ip: ip, Name: ip, Mac: mac, + Hostname: hostname, IsMaster: true, } if err := node.Add(); err != nil { @@ -240,25 +242,31 @@ func GetNodeCount(query interface{}) (int, error) { } // 节点基本信息 -func GetNodeBaseInfo() (ip string, mac string, key string, error error) { +func GetNodeBaseInfo() (ip string, mac string, hostname string, key string, error error) { ip, err := register.GetRegister().GetIp() if err != nil { debug.PrintStack() - return "", "", "", err + return "", "", "", "", err } mac, err = register.GetRegister().GetMac() if err != nil { debug.PrintStack() - return "", "", "", err + return "", "", "", "", err + } + + hostname, err = register.GetRegister().GetHostname() + if err != nil { + debug.PrintStack() + return "", "", "", "", err } key, err = register.GetRegister().GetKey() if err != nil { debug.PrintStack() - return "", "", "", err + return "", "", "", "", err } - return ip, mac, key, nil + return ip, mac, key, hostname, nil } // 根据redis的key值，重置node节点为offline diff --git a/backend/services/node.go b/backend/services/node.go index f189ddb3..f8db160a 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -23,6 +23,7 @@ type Data struct { Key string `json:"key"` Mac string `json:"mac"` Ip string `json:"ip"` + Hostname string `json:"hostname"` Master bool `json:"master"` UpdateTs time.Time `json:"update_ts"` UpdateTsUnix int64 `json:"update_ts_unix"` @@ -182,6 +183,7 @@ func UpdateNodeData() { Key: key, Mac: mac, Ip: ip, + Hostname: hostname, Master: model.IsMaster(), UpdateTs: time.Now(), UpdateTsUnix: time.Now().Unix(), @@ -199,7 +201,6 @@ func UpdateNodeData() { log.Errorf(err.Error()) return } - } func MasterNodeCallback(message redis.Message) (err error) { diff --git a/backend/services/register/register.go b/backend/services/register/register.go index bbdec2f3..8169ae65 100644 --- a/backend/services/register/register.go +++ b/backend/services/register/register.go @@ -1,10 +1,13 @@ package register import ( + "bytes" "crawlab/constants" + "fmt" "github.com/apex/log" "github.com/spf13/viper" "net" + "os/exec" "reflect" "runtime/debug" "sync" @@ -19,6 +22,8 @@ type Register interface { GetIp() (string, error) // 注册节点的mac地址 GetMac() (string, error) + // 注册节点的Hostname + GetHostname() (string, error) } // ===================== mac 地址注册 ===================== @@ -40,6 +45,10 @@ func (mac *MacRegister) GetIp() (string, error) { return getIp() } +func (mac *MacRegister) GetHostname() (string, error) { + return getHostname() +} + // ===================== ip 地址注册 ===================== type IpRegister struct { Ip string @@ -61,6 +70,33 @@ func (ip *IpRegister) GetMac() (string, error) { return getMac() } +func (ip *IpRegister) GetHostname() (string, error) { + return getHostname() +} + +// ===================== mac 地址注册 ===================== +type HostnameRegister struct{} + +func (h *HostnameRegister) GetType() string { + return "mac" +} + +func (h *HostnameRegister) GetKey() (string, error) { + return h.GetHostname() +} + +func (h *HostnameRegister) GetMac() (string, error) { + return getMac() +} + +func (h *HostnameRegister) GetIp() (string, error) { + return getIp() +} + +func (h *HostnameRegister) GetHostname() (string, error) { + return getHostname() +} + // ===================== 公共方法 ===================== // 获取本机的IP地址 // TODO: 考虑多个IP地址的情况 @@ -95,6 +131,23 @@ func getMac() (string, error) { return "", nil } +func getHostname() (string, error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("hostname") + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf(fmt.Sprintf("error: %s", stderr.String())) + debug.PrintStack() + return "", err + } + + return stdout.String(), nil +} + // ===================== 获得注册简单工厂 ===================== var register Register @@ -122,6 +175,7 @@ func GetRegister() Register { Ip: ip, } } else if registerType == constants.RegisterTypeHostname { + register = &HostnameRegister{} } log.Info("register type is :" + reflect.TypeOf(register).String()) From 1409cfb0f8b5ddcb53412e0a97e8628bbe69a5cd Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 09:31:11 +0800 Subject: [PATCH 10/12] try to fix install java error --- backend/scripts/install-java.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh index c42a2901..d99b16be 100755 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -4,7 +4,7 @@ touch /tmp/install-java.lock # install java -apt-get update && apt-get install -y default-jdk +apt-get update && apt-get install -y default-jdk --fix-missing ln -s /usr/bin/java /usr/local/bin/java # unlock From 13963528cada08e0922b04f71a70f72cb9e5645f Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 11:01:26 +0800 Subject: [PATCH 11/12] =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/utils/tour.js | 1 + frontend/src/views/node/NodeDetail.vue | 1 - frontend/src/views/schedule/ScheduleList.vue | 2 -- frontend/src/views/setting/Setting.vue | 1 - frontend/src/views/spider/SpiderDetail.vue | 2 -- frontend/src/views/spider/SpiderList.vue | 2 -- frontend/src/views/task/TaskDetail.vue | 1 - frontend/src/views/task/TaskList.vue | 1 - 8 files changed, 1 insertion(+), 10 deletions(-) diff --git a/frontend/src/utils/tour.js b/frontend/src/utils/tour.js index 33945603..7696c8ed 100644 --- a/frontend/src/utils/tour.js +++ b/frontend/src/utils/tour.js @@ -21,6 +21,7 @@ export default { startTour: (vm, tourName) => { if (localStorage.getItem('enableTutorial') === '0') return vm.$tours[tourName].start() + vm.$st.sendEv('教程', '开始', tourName) }, finishTour: (tourName) => { let data diff --git a/frontend/src/views/node/NodeDetail.vue b/frontend/src/views/node/NodeDetail.vue index 7f87a87e..bfef601b 100644 --- a/frontend/src/views/node/NodeDetail.vue +++ b/frontend/src/views/node/NodeDetail.vue @@ -125,7 +125,6 @@ export default { mounted () { if (!this.$utils.tour.isFinishedTour('node-detail')) { this.$utils.tour.startTour(this, 'node-detail') - this.$st.sendEv('教程', '开始', 'node-detail') } } } diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index adbfef56..68f8b012 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -468,7 +468,6 @@ export default { if (!this.$utils.tour.isFinishedTour('schedule-list-add')) { setTimeout(() => { this.$utils.tour.startTour(this, 'schedule-list-add') - this.$st.sendEv('教程', '开始', 'schedule-list-add') }, 500) } }, @@ -618,7 +617,6 @@ export default { if (!this.isDisabledSpiderSchedule) { if (!this.$utils.tour.isFinishedTour('schedule-list')) { this.$utils.tour.startTour(this, 'schedule-list') - this.$st.sendEv('教程', '开始', 'schedule-list') } } } diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue index 86118e71..c2c7c7f2 100644 --- a/frontend/src/views/setting/Setting.vue +++ b/frontend/src/views/setting/Setting.vue @@ -315,7 +315,6 @@ export default { mounted () { if (!this.$utils.tour.isFinishedTour('setting')) { this.$utils.tour.startTour(this, 'setting') - this.$st.sendEv('教程', '开始', 'setting') } } } diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index a69545b2..02038caf 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -209,7 +209,6 @@ export default { if (!this.$utils.tour.isFinishedTour('spider-detail-config')) { setTimeout(() => { this.$utils.tour.startTour(this, 'spider-detail-config') - this.$st.sendEv('教程', '开始', 'spider-detail-config') }, 100) } } else if (this.activeTabName === 'scrapy-settings') { @@ -261,7 +260,6 @@ export default { mounted () { if (!this.$utils.tour.isFinishedTour('spider-detail')) { this.$utils.tour.startTour(this, 'spider-detail') - this.$st.sendEv('教程', '开始', 'spider-detail') } } } diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 8e6a6ba9..10dca8c0 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -888,7 +888,6 @@ export default { setTimeout(() => { if (!this.$utils.tour.isFinishedTour('spider-list-add')) { this.$utils.tour.startTour(this, 'spider-list-add') - this.$st.sendEv('教程', '开始', 'spider-list-add') } }, 300) }, @@ -1242,7 +1241,6 @@ export default { if (!this.$utils.tour.isFinishedTour('spider-list')) { this.$utils.tour.startTour(this, 'spider-list') - this.$st.sendEv('教程', '开始', 'spider-list') } }, destroyed () { diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index 422dc088..0ecbdf9e 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -203,7 +203,6 @@ export default { mounted () { if (!this.$utils.tour.isFinishedTour('task-detail')) { this.$utils.tour.startTour(this, 'task-detail') - this.$st.sendEv('教程', '开始', 'task-detail') } }, destroyed () { diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index 3cd564d9..1425e2da 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -413,7 +413,6 @@ export default { if (!this.$utils.tour.isFinishedTour('task-list')) { this.$utils.tour.startTour(this, 'task-list') - this.$st.sendEv('教程', '开始', 'task-list') } }, destroyed () { From f0382ab7680bfd19c4b46e3a416fe0d7f4fe4cc4 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 11:01:58 +0800 Subject: [PATCH 12/12] =?UTF-8?q?=E6=9A=82=E6=97=B6=E5=85=B3=E9=97=ADjava?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/system.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/system.go b/backend/services/system.go index b2884378..285cad5a 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -67,7 +67,7 @@ func GetLangList(nodeId string) []entity.Lang { list := []entity.Lang{ {Name: "Python", ExecutableName: "python", ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, DepExecutablePath: "/usr/local/bin/pip"}, {Name: "Node.js", ExecutableName: "node", ExecutablePaths: []string{"/usr/bin/node", "/usr/local/bin/node"}, DepExecutablePath: "/usr/local/bin/npm"}, - {Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}}, + //{Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}}, } for i, lang := range list { list[i].Installed = IsInstalledLang(nodeId, lang)