From c036c0eeb345a78bfeb88f592d432098c0393f51 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 11 Mar 2020 18:10:18 +0800 Subject: [PATCH 1/6] fixed k8s yaml --- k8s/crawlab-master.yaml | 2 +- k8s/crawlab-worker.yaml | 2 +- k8s/ns.yaml | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 k8s/ns.yaml diff --git a/k8s/crawlab-master.yaml b/k8s/crawlab-master.yaml index d8dff5be..60fc76bd 100644 --- a/k8s/crawlab-master.yaml +++ b/k8s/crawlab-master.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: crawlab - namespace: crawlab-master + namespace: crawlab spec: ports: - port: 8080 diff --git a/k8s/crawlab-worker.yaml b/k8s/crawlab-worker.yaml index 4f6e9002..2ee052d0 100644 --- a/k8s/crawlab-worker.yaml +++ b/k8s/crawlab-worker.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: crawlab-worker - namespace: crawlab-master + namespace: crawlab spec: serviceName: crawlab-worker replicas: 2 diff --git a/k8s/ns.yaml b/k8s/ns.yaml new file mode 100644 index 00000000..ff3d460b --- /dev/null +++ b/k8s/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: crawlab \ No newline at end of file From 59a5888b2c4e027e11a43f0b0d25cfcb6dc09c7a Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 11 Mar 2020 18:15:25 +0800 Subject: [PATCH 2/6] fixed k8s yaml --- k8s/crawlab-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/crawlab-master.yaml b/k8s/crawlab-master.yaml index 60fc76bd..b2274684 100644 --- a/k8s/crawlab-master.yaml +++ b/k8s/crawlab-master.yaml @@ -16,7 +16,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: crawlab-master - namespace: crawlab-master + namespace: crawlab spec: serviceName: crawlab-master selector: From 15e4b1029394da8d01d1cb7b7f184933f2875e26 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 11 Mar 2020 20:36:38 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/Settings/GitSettings.vue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/Settings/GitSettings.vue b/frontend/src/components/Settings/GitSettings.vue index 048f2df6..c80e5427 100644 --- a/frontend/src/components/Settings/GitSettings.vue +++ b/frontend/src/components/Settings/GitSettings.vue @@ -2,6 +2,7 @@ Date: Wed, 11 Mar 2020 22:58:45 +0800 Subject: [PATCH 4/6] updated README --- README-zh.md | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README-zh.md b/README-zh.md index 0556532a..1adf3dc7 100644 --- a/README-zh.md +++ b/README-zh.md @@ -37,7 +37,7 @@ 三种方式: 1. [Docker](http://docs.crawlab.cn/Installation/Docker.html)（推荐） 2. [直接部署](http://docs.crawlab.cn/Installation/Direct.html)（了解内核） -3. [Kubernetes](https://juejin.im/post/5e0a02d851882549884c27ad) (多节点部署) +3. [Kubernetes](http://docs.crawlab.cn/Installation/Kubernetes.html) (多节点部署) ### 要求（Docker） - Docker 18.03+ diff --git a/README.md b/README.md index 8224cc77..25a16c8e 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,10 @@ Golang-based distributed web crawler management platform, supporting various lan ## Installation -Two methods: +Three methods: 1. [Docker](http://docs.crawlab.cn/Installation/Docker.html) (Recommended) 2. [Direct Deploy](http://docs.crawlab.cn/Installation/Direct.html) (Check Internal Kernel) -3. [Kubernetes](https://juejin.im/post/5e0a02d851882549884c27ad) (Multi-Node Deployment) +3. [Kubernetes](http://docs.crawlab.cn/Installation/Kubernetes.html) (Multi-Node Deployment) ### Pre-requisite (Docker) - Docker 18.03+ From 3c7bd0bff0360e80a636133a258a47500dcd5e9f Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 12 Mar 2020 08:52:16 +0800 Subject: [PATCH 5/6] updated k8s yaml --- devops/master/crawlab-master.yaml | 57 +++++++++++++++++++++++++++++++ devops/master/crawlab-worker.yaml | 35 +++++++++++++++++++ devops/master/mongo-pv.yaml | 28 +++++++++++++++ devops/master/mongo.yaml | 41 ++++++++++++++++++++++ devops/master/ns.yaml | 4 +++ devops/master/redis.yaml | 34 ++++++++++++++++++ 6 files changed, 199 insertions(+) create mode 100644 devops/master/crawlab-master.yaml create mode 100644 devops/master/crawlab-worker.yaml create mode 100644 devops/master/mongo-pv.yaml create mode 100644 devops/master/mongo.yaml create mode 100644 devops/master/ns.yaml create mode 100644 devops/master/redis.yaml diff --git a/devops/master/crawlab-master.yaml b/devops/master/crawlab-master.yaml new file mode 100644 index 00000000..21a06069 --- /dev/null +++ b/devops/master/crawlab-master.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Service +metadata: + name: crawlab + namespace: crawlab +spec: + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30088 + selector: + app: crawlab-master + type: NodePort +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: crawlab-master + namespace: crawlab +spec: + serviceName: crawlab-master + selector: + matchLabels: + app: crawlab-master + template: + metadata: + labels: + app: crawlab-master + spec: + containers: + - image: tikazyq/crawlab:latest + imagePullPolicy: Always + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "Y" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + - name: CRAWLAB_SETTING_ALLOWREGISTER + value: "Y" + - name: CRAWLAB_SERVER_LANG_NODE + value: "N" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "N" + - name: CRAWLAB_SERVER_LANG_DOTNET + value: "N" + - name: CRAWLAB_SERVER_REGISTER_TYPE + value: "hostname" + - name: CRAWLAB_SERVER_DEMOSPIDERS + value: "Y" + - name: CRAWLAB_SERVER_ENABLETUTORIAL + value: "Y" + ports: + - containerPort: 8080 + name: crawlab \ No newline at end of file diff --git a/devops/master/crawlab-worker.yaml b/devops/master/crawlab-worker.yaml new file mode 100644 index 00000000..2ee052d0 --- /dev/null +++ b/devops/master/crawlab-worker.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: crawlab-worker + namespace: crawlab +spec: + serviceName: crawlab-worker + replicas: 2 + selector: + matchLabels: + app: crawlab-worker + template: + metadata: + labels: + app: crawlab-worker + spec: + containers: + - image: tikazyq/crawlab:latest + imagePullPolicy: Always + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "N" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" + - name: CRAWLAB_SERVER_LANG_DOTNET + value: "Y" + - name: CRAWLAB_SERVER_REGISTER_TYPE + value: "hostname" diff --git a/devops/master/mongo-pv.yaml b/devops/master/mongo-pv.yaml new file mode 100644 index 00000000..ad914131 --- /dev/null +++ b/devops/master/mongo-pv.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: mongo-pv-volume + namespace: crawlab + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data/k8s/mongodb/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-pv-claim + namespace: crawlab +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi \ No newline at end of file diff --git a/devops/master/mongo.yaml b/devops/master/mongo.yaml new file mode 100644 index 00000000..9b5bd3b6 --- /dev/null +++ b/devops/master/mongo.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongo + namespace: crawlab +spec: + ports: + - port: 27017 + selector: + app: mongo + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: mongo + namespace: crawlab +spec: + selector: + matchLabels: + app: mongo + strategy: + type: Recreate + template: + metadata: + labels: + app: mongo + spec: + containers: + - image: mongo:4 + name: mongo + ports: + - containerPort: 27017 + name: mongo + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + volumes: + - name: mongo-persistent-storage + persistentVolumeClaim: + claimName: mongo-pv-claim \ No newline at end of file diff --git a/devops/master/ns.yaml b/devops/master/ns.yaml new file mode 100644 index 00000000..ff3d460b --- /dev/null +++ b/devops/master/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: crawlab \ No newline at end of file diff --git a/devops/master/redis.yaml b/devops/master/redis.yaml new file mode 100644 index 00000000..49df8dde --- /dev/null +++ b/devops/master/redis.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: crawlab +spec: + ports: + - port: 6379 + selector: + app: redis + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: redis + namespace: crawlab +spec: + selector: + matchLabels: + app: redis + strategy: + type: Recreate + template: + metadata: + labels: + app: redis + spec: + containers: + - image: redis + name: redis + ports: + - containerPort: 6379 + name: redis \ No newline at end of file From 19dda39931ae014736ad29f9b916444c88428430 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 12 Mar 2020 09:00:50 +0800 Subject: [PATCH 6/6] updated k8s yaml --- devops/master/crawlab-master.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/devops/master/crawlab-master.yaml b/devops/master/crawlab-master.yaml index 21a06069..6d58ff20 100644 --- a/devops/master/crawlab-master.yaml +++ b/devops/master/crawlab-master.yaml @@ -38,8 +38,6 @@ spec: value: "mongo" - name: CRAWLAB_REDIS_ADDRESS value: "redis" - - name: CRAWLAB_SETTING_ALLOWREGISTER - value: "Y" - name: CRAWLAB_SERVER_LANG_NODE value: "N" - name: CRAWLAB_SERVER_LANG_JAVA @@ -48,9 +46,11 @@ spec: value: "N" - name: CRAWLAB_SERVER_REGISTER_TYPE value: "hostname" - - name: CRAWLAB_SERVER_DEMOSPIDERS + - name: CRAWLAB_SETTING_ALLOWREGISTER value: "Y" - - name: CRAWLAB_SERVER_ENABLETUTORIAL + - name: CRAWLAB_SETTING_ENABLETUTORIAL + value: "Y" + - name: CRAWLAB_SETTING_DEMOSPIDERS value: "Y" ports: - containerPort: 8080