From 21943eb05fc0804f7b6773249b9487b22ee025bc Mon Sep 17 00:00:00 2001 From: hantmac Date: Mon, 2 Mar 2020 10:45:29 +0800 Subject: [PATCH 01/15] bug fix issue 603 --- backend/routes/spider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 1436e744..d195a5e7 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -399,7 +399,7 @@ func UploadSpider(c *gin.Context) { } // 更新file_id spider.FileId = fid - if err := spider.Add(); err != nil { + if err := spider.Save(); err != nil { log.Error("add spider error: " + err.Error()) debug.PrintStack() HandleError(http.StatusInternalServerError, c, err) From 89d1846510879978cd12940a84879fd6f7551e7a Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 10:36:58 +0800 Subject: [PATCH 02/15] added java env --- backend/entity/system.go | 10 +++++----- backend/scripts/install-java.sh | 11 ++++++++++ backend/scripts/install-nodejs.sh | 6 ++++++ backend/services/system.go | 12 ++++++----- docker-compose.yml | 1 + docker_init.sh | 9 ++++++++- .../src/components/Node/NodeInstallation.vue | 2 ++ .../Node/NodeInstallationMatrix.vue | 20 +++++++++++++++++++ frontend/src/views/node/NodeList.vue | 6 +++++- 9 files changed, 65 insertions(+), 12 deletions(-) create mode 100644 backend/scripts/install-java.sh create mode 100644 frontend/src/components/Node/NodeInstallationMatrix.vue diff --git a/backend/entity/system.go b/backend/entity/system.go index ac3e9dec..4c15cc54 100644 --- a/backend/entity/system.go +++ b/backend/entity/system.go @@ -15,11 +15,11 @@ type Executable struct { } type Lang struct { - Name string `json:"name"` - ExecutableName string `json:"executable_name"` - ExecutablePath string `json:"executable_path"` - DepExecutablePath string `json:"dep_executable_path"` - Installed bool `json:"installed"` + Name string `json:"name"` + ExecutableName string `json:"executable_name"` + ExecutablePaths []string `json:"executable_paths"` + DepExecutablePath string `json:"dep_executable_path"` + Installed bool `json:"installed"` } type Dependency struct { diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh new file mode 100644 index 00000000..e98de224 --- /dev/null +++ b/backend/scripts/install-java.sh @@ -0,0 +1,11 @@ +#!/bin/env bash + +# lock +touch /tmp/install-java.lock + +# install java +apt-get install -y default-jdk +ln -s /usr/bin/java /usr/local/bin/java + +# unlock +rm /tmp/install-java.lock diff --git a/backend/scripts/install-nodejs.sh b/backend/scripts/install-nodejs.sh index 376a2a3f..eada900a 100644 --- a/backend/scripts/install-nodejs.sh +++ b/backend/scripts/install-nodejs.sh @@ -1,5 +1,8 @@ #!/bin/env bash +# lock +touch /tmp/install-nodejs.lock + # install nvm BASE_DIR=`dirname $0` /bin/bash ${BASE_DIR}/install-nvm.sh @@ -33,3 +36,6 @@ apt-get update && apt-get install -y --no-install-recommends gconf-service libas PUPPETEER_DOWNLOAD_HOST=https://npm.taobao.org/mirrors npm config set puppeteer_download_host=https://npm.taobao.org/mirrors npm install puppeteer-chromium-resolver crawlab-sdk -g --unsafe-perm=true --registry=https://registry.npm.taobao.org + +# unlock +rm /tmp/install-nodejs.lock diff --git a/backend/services/system.go b/backend/services/system.go index 6181afee..b2884378 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -65,9 +65,9 @@ func GetSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) { // 获取语言列表 func GetLangList(nodeId string) []entity.Lang { list := []entity.Lang{ - {Name: "Python", ExecutableName: "python", ExecutablePath: "/usr/local/bin/python", DepExecutablePath: "/usr/local/bin/pip"}, - {Name: "Node.js", ExecutableName: "node", ExecutablePath: "/usr/local/bin/node", DepExecutablePath: "/usr/local/bin/npm"}, - //{Name: "Java", ExecutableName: "java", ExecutablePath: "/usr/local/bin/java"}, + {Name: "Python", ExecutableName: "python", ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, DepExecutablePath: "/usr/local/bin/pip"}, + {Name: "Node.js", ExecutableName: "node", ExecutablePaths: []string{"/usr/bin/node", "/usr/local/bin/node"}, DepExecutablePath: "/usr/local/bin/npm"}, + {Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}}, } for i, lang := range list { list[i].Installed = IsInstalledLang(nodeId, lang) @@ -93,8 +93,10 @@ func IsInstalledLang(nodeId string, lang entity.Lang) bool { return false } for _, exec := range sysInfo.Executables { - if exec.Path == lang.ExecutablePath { - return true + for _, path := range lang.ExecutablePaths { + if exec.Path == path { + return true + } } } return false diff --git a/docker-compose.yml b/docker-compose.yml index affdf7bb..26cbb989 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: # CRAWLAB_SERVER_REGISTER_IP: "127.0.0.1" # node register ip 节点注册IP. 节点唯一识别号，只有当 CRAWLAB_SERVER_REGISTER_TYPE 为 "ip" 时才生效 # CRAWLAB_TASK_WORKERS: 4 # number of task executors 任务执行器个数（并行执行任务数） # CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境 + # CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境 # CRAWLAB_SETTING_ALLOWREGISTER: "N" # whether to allow user registration 是否允许用户注册 # CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程 # CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址 diff --git a/docker_init.sh b/docker_init.sh index 4baee046..9c51fa83 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -26,7 +26,14 @@ service nginx start if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; then echo "installing node.js" - /bin/sh /app/backend/scripts/install-nodejs.sh + /bin/sh /app/backend/scripts/install-nodejs.sh & +fi + +# install languages: Java +if [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ]; +then + echo "installing java" + /bin/sh /app/backend/scripts/install-java.sh & fi # generate ssh diff --git a/frontend/src/components/Node/NodeInstallation.vue b/frontend/src/components/Node/NodeInstallation.vue index 2d466ab7..68eb14c7 100644 --- a/frontend/src/components/Node/NodeInstallation.vue +++ b/frontend/src/components/Node/NodeInstallation.vue @@ -152,6 +152,7 @@ export default { methods: { async getDepList () { this.loading = true + this.depList = [] const res = await this.$request.get(`/nodes/${this.nodeForm._id}/deps`, { lang: this.activeLang.executable_name, dep_name: this.depName @@ -175,6 +176,7 @@ export default { }, async getInstalledDepList () { this.loading = true + this.installedDepList = [] const res = await this.$request.get(`/nodes/${this.nodeForm._id}/deps/installed`, { lang: this.activeLang.executable_name }) diff --git a/frontend/src/components/Node/NodeInstallationMatrix.vue b/frontend/src/components/Node/NodeInstallationMatrix.vue new file mode 100644 index 00000000..c08dcaeb --- /dev/null +++ b/frontend/src/components/Node/NodeInstallationMatrix.vue @@ -0,0 +1,20 @@ + + + + + diff --git a/frontend/src/views/node/NodeList.vue b/frontend/src/views/node/NodeList.vue index 2b721950..a3320f8d 100644 --- a/frontend/src/views/node/NodeList.vue +++ b/frontend/src/views/node/NodeList.vue @@ -142,6 +142,9 @@ + + + @@ -153,10 +156,11 @@ import { } from 'vuex' import 'github-markdown-css/github-markdown.css' import NodeNetwork from '../../components/Node/NodeNetwork' +import NodeInstallationMatrix from '../../components/Node/NodeInstallationMatrix' export default { name: 'NodeList', - components: { NodeNetwork }, + components: { NodeInstallationMatrix, NodeNetwork }, data () { return { pagination: { From 2af88a99ace0451e4205d5dab68d21564410920a Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 11:32:29 +0800 Subject: [PATCH 03/15] code cleanup --- devops/release/crawlab-master.yaml | 45 ++++++++++++ devops/release/crawlab-worker.yaml | 27 +++++++ devops/release/mongo-pv.yaml | 28 ++++++++ devops/release/mongo.yaml | 41 +++++++++++ devops/release/ns.yaml | 4 ++ devops/release/redis.yaml | 34 +++++++++ docker/Dockerfile.frontend.alpine | 27 ------- docker/Dockerfile.master.alpine | 86 ----------------------- docker/Dockerfile.worker.alpine | 43 ------------ examples/README.md | 7 -- examples/frontend/Dockerfile | 30 -------- examples/frontend/README.md | 5 -- examples/frontend/crawlab.conf | 13 ---- examples/frontend/docker_init.sh | 23 ------ examples/master/Dockerfile | 20 ------ examples/master/README.md | 8 --- examples/master/config.yml | 32 --------- examples/run_docker_master.sh | 10 --- examples/run_docker_worker.sh | 6 -- examples/worker/python/Dockerfile | 24 ------- examples/worker/python/README.md | 26 ------- examples/worker/python/conf/config.yml | 25 ------- examples/worker/python/docker-compose.yml | 18 ----- examples/worker/python/requirements.txt | 7 -- jenkins/develop/docker-compose.yaml | 32 --------- jenkins/master/docker-compose.yaml | 51 -------------- 26 files changed, 179 insertions(+), 493 deletions(-) create mode 100644 devops/release/crawlab-master.yaml create mode 100644 devops/release/crawlab-worker.yaml create mode 100644 devops/release/mongo-pv.yaml create mode 100644 devops/release/mongo.yaml create mode 100644 devops/release/ns.yaml create mode 100644 devops/release/redis.yaml delete mode 100644 docker/Dockerfile.frontend.alpine delete mode 100644 docker/Dockerfile.master.alpine delete mode 100644 docker/Dockerfile.worker.alpine delete mode 100644 examples/README.md delete mode 100644 examples/frontend/Dockerfile delete mode 100644 examples/frontend/README.md delete mode 100644 examples/frontend/crawlab.conf delete mode 100755 examples/frontend/docker_init.sh delete mode 100644 examples/master/Dockerfile delete mode 100644 examples/master/README.md delete mode 100644 examples/master/config.yml delete mode 100644 examples/run_docker_master.sh delete mode 100644 examples/run_docker_worker.sh delete mode 100644 examples/worker/python/Dockerfile delete mode 100644 examples/worker/python/README.md delete mode 100644 examples/worker/python/conf/config.yml delete mode 100644 examples/worker/python/docker-compose.yml delete mode 100644 examples/worker/python/requirements.txt delete mode 100644 jenkins/develop/docker-compose.yaml delete mode 100644 jenkins/master/docker-compose.yaml diff --git a/devops/release/crawlab-master.yaml b/devops/release/crawlab-master.yaml new file mode 100644 index 00000000..e886fdd3 --- /dev/null +++ b/devops/release/crawlab-master.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Service +metadata: + name: crawlab + namespace: crawlab-release +spec: + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30098 + selector: + app: crawlab-master + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-master + namespace: crawlab-release +spec: + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-master + template: + metadata: + labels: + app: crawlab-master + spec: + containers: + - image: tikazyq/crawlab:release + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "Y" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + - name: CRAWLAB_SETTING_ALLOWREGISTER + value: "Y" + ports: + - containerPort: 8080 + name: crawlab \ No newline at end of file diff --git a/devops/release/crawlab-worker.yaml b/devops/release/crawlab-worker.yaml new file mode 100644 index 00000000..5bc2f396 --- /dev/null +++ b/devops/release/crawlab-worker.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-worker + namespace: crawlab-release +spec: + replicas: 2 + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-worker + template: + metadata: + labels: + app: crawlab-worker + spec: + containers: + - image: tikazyq/crawlab:release + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "N" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" \ No newline at end of file diff --git a/devops/release/mongo-pv.yaml b/devops/release/mongo-pv.yaml new file mode 100644 index 00000000..44503a01 --- /dev/null +++ b/devops/release/mongo-pv.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: mongo-pv-volume-release + namespace: crawlab-release + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 5Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data/crawlab-release/mongodb/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-pv-claim-release + namespace: crawlab-release +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi \ No newline at end of file diff --git a/devops/release/mongo.yaml b/devops/release/mongo.yaml new file mode 100644 index 00000000..d169174a --- /dev/null +++ b/devops/release/mongo.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongo + namespace: crawlab-release +spec: + ports: + - port: 27017 + selector: + app: mongo + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: mongo + namespace: crawlab-release +spec: + selector: + matchLabels: + app: mongo + strategy: + type: Recreate + template: + metadata: + labels: + app: mongo + spec: + containers: + - image: mongo:4 + name: mongo + ports: + - containerPort: 27017 + name: mongo + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + volumes: + - name: mongo-persistent-storage + persistentVolumeClaim: + claimName: mongo-pv-claim-release \ No newline at end of file diff --git a/devops/release/ns.yaml b/devops/release/ns.yaml new file mode 100644 index 00000000..ee77dde9 --- /dev/null +++ b/devops/release/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: crawlab-release \ No newline at end of file diff --git a/devops/release/redis.yaml b/devops/release/redis.yaml new file mode 100644 index 00000000..459a033a --- /dev/null +++ b/devops/release/redis.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: crawlab-release +spec: + ports: + - port: 6379 + selector: + app: redis + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: redis + namespace: crawlab-release +spec: + selector: + matchLabels: + app: redis + strategy: + type: Recreate + template: + metadata: + labels: + app: redis + spec: + containers: + - image: redis + name: redis + ports: + - containerPort: 6379 + name: redis \ No newline at end of file diff --git a/docker/Dockerfile.frontend.alpine b/docker/Dockerfile.frontend.alpine deleted file mode 100644 index f809ee7c..00000000 --- a/docker/Dockerfile.frontend.alpine +++ /dev/null @@ -1,27 +0,0 @@ -FROM node:8.16.0 AS frontend-build - -ADD ./frontend /app -WORKDIR /app - -# install frontend -RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org - -RUN npm run build:prod - -FROM alpine:latest - -# 安装nginx -RUN mkdir /run/nginx && apk add nginx - -# 拷贝编译文件 -COPY --from=frontend-build /app/dist /app/dist - -# 拷贝nginx代理文件 -COPY crawlab.conf /etc/nginx/conf.d - -# 拷贝执行脚本 -COPY docker_init.sh /app/docker_init.sh - -EXPOSE 8080 - -CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.master.alpine b/docker/Dockerfile.master.alpine deleted file mode 100644 index b9dbb742..00000000 --- a/docker/Dockerfile.master.alpine +++ /dev/null @@ -1,86 +0,0 @@ -FROM golang:1.12-alpine AS backend-build - -WORKDIR /go/src/app -COPY ./backend . - -ENV GO111MODULE on -ENV GOPROXY https://mirrors.aliyun.com/goproxy/ - -RUN go install -v ./... - -FROM node:lts-alpine AS frontend-build - -ARG NPM_DISABLE_SAFE_PERM=false -RUN if [ ${NPM_DISABLE_SAFE_PERM} = true ]; then \ - # run the install - echo "info: use npm unsafe-perm mode" \ - && npm config set unsafe-perm true \ -;fi - - -ADD ./frontend /app -WORKDIR /app - -# install frontend -RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org - -RUN npm run build:prod - -# images -FROM python:alpine - -ADD . /app - - -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories - -# install packages -RUN apk update && apk add --no-cache --virtual .build-deps \ - gcc \ - linux-headers \ - musl-dev \ - libffi-dev \ - libxml2-dev \ - libxslt-dev \ - openssl-dev - -# install backend -RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple - -# copy backend files -COPY --from=backend-build /go/src/app/conf ./conf -COPY --from=backend-build /go/bin/crawlab /usr/local/bin - -# install nginx -RUN apk add --no-cache nginx openrc - -RUN apk del .build-deps -# copy frontend files -COPY --from=frontend-build /app/dist /app/dist -COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d - -VOLUME [ “/sys/fs/cgroup” ] - -RUN sed -i 's/#rc_sys=""/rc_sys="lxc"/g' /etc/rc.conf && \ - echo 'rc_provide="loopback net"' >> /etc/rc.conf && \ - sed -i 's/^#\(rc_logger="YES"\)$/\1/' /etc/rc.conf && \ - sed -i '/tty/d' /etc/inittab && \ - sed -i 's/hostname $opts/# hostname $opts/g' /etc/init.d/hostname && \ - sed -i 's/mount -t tmpfs/# mount -t tmpfs/g' /lib/rc/sh/init.sh && \ - sed -i 's/cgroup_add_service /# cgroup_add_service /g' /lib/rc/sh/openrc-run.sh && \ - rm -rf /var/cache/apk/* && \ - mkdir -p /run/openrc && \ - touch /run/openrc/softlevel && \ - /sbin/openrc - -# working directory -WORKDIR /app/backend -ENV PYTHONIOENCODING utf-8 -# frontend port -EXPOSE 8080 - -# backend port -EXPOSE 8000 - -# start backend -CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.worker.alpine b/docker/Dockerfile.worker.alpine deleted file mode 100644 index 388125a2..00000000 --- a/docker/Dockerfile.worker.alpine +++ /dev/null @@ -1,43 +0,0 @@ -FROM golang:1.12-alpine AS backend-build - -WORKDIR /go/src/app -COPY ./backend . - -ENV GO111MODULE on -ENV GOPROXY https://mirrors.aliyun.com/goproxy/ - -RUN go install -v ./... - -# images -FROM python:alpine - -ADD . /app - -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories - -RUN apk update && apk add --no-cache --virtual .build-deps \ - gcc \ - linux-headers \ - musl-dev \ - libffi-dev \ - libxml2-dev \ - libxslt-dev \ - openssl-dev - -# install backend -RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple - -# copy backend files -COPY --from=backend-build /go/src/app/conf ./conf -COPY --from=backend-build /go/bin/crawlab /usr/local/bin - -RUN apk del .build-deps - -# working directory -WORKDIR /app/backend -ENV PYTHONIOENCODING utf-8 -# backend port -EXPOSE 8000 - -# start backend -CMD crawlab \ No newline at end of file diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index a42197c5..00000000 --- a/examples/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Examples - -* frontend 前端镜像制作 -* master Master节点镜像制作 -* worker Worker节点镜像制作 -* run_docker_master.sh 运行Master节点示例脚本 -* run_docker_worker.sh 运行Worker节点示例脚本 \ No newline at end of file diff --git a/examples/frontend/Dockerfile b/examples/frontend/Dockerfile deleted file mode 100644 index 9ce88b35..00000000 --- a/examples/frontend/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM node:8.16.0-alpine AS build - -ADD ./frontend /app -WORKDIR /app - -# install frontend -RUN npm install -g yarn && yarn install - -RUN npm run build:prod - -FROM alpine:latest - -# 安装nginx -RUN mkdir /run/nginx && apk add nginx - -# 拷贝编译文件 -COPY --from=build /app/dist /app/dist - -# 拷贝nginx代理文件 -COPY crawlab.conf /etc/nginx/conf.d - -# 拷贝执行脚本 -COPY docker_init.sh /app/docker_init.sh - -# 定义后端API脚本 -ENV CRAWLAB_API_ADDRESS http://localhost:8000 - -EXPOSE 8080 - -CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/examples/frontend/README.md b/examples/frontend/README.md deleted file mode 100644 index efe9493f..00000000 --- a/examples/frontend/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# 前端镜像制作 - -前端需要手动build拿到编译后的文件，然后放入此目录进行镜像构建。 - -容器运行的时候需要指定`CRAWLAB_API_ADDRESS`环境变量，用于访问后端API接口 \ No newline at end of file diff --git a/examples/frontend/crawlab.conf b/examples/frontend/crawlab.conf deleted file mode 100644 index bf8270e5..00000000 --- a/examples/frontend/crawlab.conf +++ /dev/null @@ -1,13 +0,0 @@ -server { - gzip on; - gzip_min_length 1k; - gzip_buffers 4 16k; - #gzip_http_version 1.0; - gzip_comp_level 2; - gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; - gzip_vary off; - gzip_disable "MSIE [1-6]\."; - listen 8080; - root /app/dist; - index index.html; -} diff --git a/examples/frontend/docker_init.sh b/examples/frontend/docker_init.sh deleted file mode 100755 index cfd53442..00000000 --- a/examples/frontend/docker_init.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -# replace default api path to new one -if [ "${CRAWLAB_API_ADDRESS}" = "" ]; -then - : -else - jspath=`ls /app/dist/js/app.*.js` - sed -i "s?localhost:8000?${CRAWLAB_API_ADDRESS}?g" ${jspath} -fi - -# replace base url -if [ "${CRAWLAB_BASE_URL}" = "" ]; -then - : -else - indexpath=/app/dist/index.html - sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath} - sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath} -fi - -# start nginx -nginx -g 'daemon off;' \ No newline at end of file diff --git a/examples/master/Dockerfile b/examples/master/Dockerfile deleted file mode 100644 index 7f12c733..00000000 --- a/examples/master/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM alpine:latest - -# 配置工作目录 -WORKDIR /opt/crawlab - -# 拷贝配置文件 -COPY config.yml /opt/crawlab/conf/config.yml - -# 拷贝可执行文件 -COPY crawlab /usr/local/bin - -# 创建spiders文件用于存放爬虫， 授权可执行文件 -RUN mkdir -p /opt/crawlab/spiders && chmod +x /usr/local/bin/crawlab - -# 指定为Master节点 -ENV CRAWLAB_SERVER_MASTER Y - -EXPOSE 8000 - -CMD ["crawlab"] \ No newline at end of file diff --git a/examples/master/README.md b/examples/master/README.md deleted file mode 100644 index df68e725..00000000 --- a/examples/master/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Master 节点镜像制作 - -在Dockerfile里面的二进制包，需要手动在源码目录下进行构建然后再放进来。 - -## Linux 二进制包构建 -``` -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o crawlab main.go -``` \ No newline at end of file diff --git a/examples/master/config.yml b/examples/master/config.yml deleted file mode 100644 index 3e692679..00000000 --- a/examples/master/config.yml +++ /dev/null @@ -1,32 +0,0 @@ -api: - address: "localhost:8000" -mongo: - host: "192.168.235.26" - port: 27017 - db: crawlab_local - username: "root" - password: "example" - authSource: "admin" -redis: - address: 192.168.235.0 - password: redis-1.0 - database: 29 - port: 16379 -log: - level: info - path: "/logs/crawlab" -server: - host: 0.0.0.0 - port: 8000 - master: "Y" - secret: "crawlab" - register: - # mac 或者 ip，如果是ip，则需要手动指定IP - type: "mac" - ip: "192.168.0.104" -spider: - path: "/spiders" -task: - workers: 4 -other: - tmppath: "/tmp" diff --git a/examples/run_docker_master.sh b/examples/run_docker_master.sh deleted file mode 100644 index a6c18419..00000000 --- a/examples/run_docker_master.sh +++ /dev/null @@ -1,10 +0,0 @@ -docker run -d --restart always --name crawlab \ - -e CRAWLAB_REDIS_ADDRESS=192.168.99.1:6379 \ - -e CRAWLAB_MONGO_HOST=192.168.99.1 \ - -e CRAWLAB_SERVER_MASTER=Y \ - -e CRAWLAB_API_ADDRESS=192.168.99.100:8000 \ - -e CRAWLAB_SPIDER_PATH=/app/spiders \ - -p 8080:8080 \ - -p 8000:8000 \ - -v /var/logs/crawlab:/var/logs/crawlab \ - tikazyq/crawlab:0.3.0 \ No newline at end of file diff --git a/examples/run_docker_worker.sh b/examples/run_docker_worker.sh deleted file mode 100644 index 6592c2d6..00000000 --- a/examples/run_docker_worker.sh +++ /dev/null @@ -1,6 +0,0 @@ -docker run --restart always --name crawlab \ - -e CRAWLAB_REDIS_ADDRESS=192.168.99.1:6379 \ - -e CRAWLAB_MONGO_HOST=192.168.99.1 \ - -e CRAWLAB_SERVER_MASTER=N \ - -v /var/logs/crawlab:/var/logs/crawlab \ - tikazyq/crawlab:0.3.0 \ No newline at end of file diff --git a/examples/worker/python/Dockerfile b/examples/worker/python/Dockerfile deleted file mode 100644 index a3907914..00000000 --- a/examples/worker/python/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:latest - -ENV DEBIAN_FRONTEND=noninteractive - -# 添加依赖描述文件 -ADD requirements.txt /opt/crawlab/ -# 添加二进制包 -ADD crawlab /usr/local/bin/ - -RUN chmod +x /usr/local/bin/crawlab - -# 安装基本环境 -RUN apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp python3 python3-pip \ - && apt-get clean \ - && ln -s /usr/bin/pip3 /usr/local/bin/pip \ - && ln -s /usr/bin/python3 /usr/local/bin/python - -# 安装依赖 -RUN pip install -r /opt/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - -WORKDIR /opt/crawlab - -ENTRYPOINT ["crawlab"] \ No newline at end of file diff --git a/examples/worker/python/README.md b/examples/worker/python/README.md deleted file mode 100644 index a0c2b6ed..00000000 --- a/examples/worker/python/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# 本地开发环境worker节点制作 -由于master和worker节点的存储信息是在redis上，并且使用节点所在的mac地址作为key，所以在开发本地需要启动master和worker节点会比较麻烦。 -这里是一个运行worker节点的一个例子。 - -基本思路是worker节点所需的依赖制作成一个镜像，然后把crawlab编译成二进制包，接着把配置文件和二进制包通过volumes的形式挂载到容器内部。 -这样就可以正常的运行worker节点了。之后对于容器编排的worker节点，可以直接把该镜像当成worker节点的基础镜像。 - -### 制作二进制包 -在`backend`目录下执行以下命令，生成二进制包 -``` -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o crawlab main.go -``` - - -### 构建worker镜像 -``` -docker build -t crawlab:worker . -``` - -### 运行worker节点 -``` -docker-compose up -d -``` - -如果在多台服务器使用`docker-compose.yml`进行编排，可能出现节点注册不上的问题，因为mac地址冲突了。 -可以使用`networks`定义当前节点的IP段，这样就可以正常注册到redis \ No newline at end of file diff --git a/examples/worker/python/conf/config.yml b/examples/worker/python/conf/config.yml deleted file mode 100644 index 7ea87252..00000000 --- a/examples/worker/python/conf/config.yml +++ /dev/null @@ -1,25 +0,0 @@ -mongo: - host: 127.0.0.1 - port: 27017 - db: crawlab_test - username: "" - password: "" -redis: - address: 127.0.0.1 - password: "" - database: 1 - port: 6379 -log: - level: info - path: "/opt/crawlab/logs" -server: - host: 0.0.0.0 - port: 8000 - master: "N" - secret: "crawlab" -spider: - path: "/opt/crawlab/spiders" -task: - workers: 4 -other: - tmppath: "/tmp" diff --git a/examples/worker/python/docker-compose.yml b/examples/worker/python/docker-compose.yml deleted file mode 100644 index 63d9c90c..00000000 --- a/examples/worker/python/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3' -services: - worker: - image: crawlab:worker - container_name: crawlab-worker - volumes: - - $PWD/conf/config.yml:/opt/crawlab/conf/config.yml - # 二进制包使用源码生成 - - $PWD/crawlab:/usr/local/bin/crawlab - networks: - - crawlabnet - -networks: - crawlabnet: - ipam: - driver: default - config: - - subnet: 172.30.0.0/16 \ No newline at end of file diff --git a/examples/worker/python/requirements.txt b/examples/worker/python/requirements.txt deleted file mode 100644 index 70cd908b..00000000 --- a/examples/worker/python/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -geohash2==1.1 -Scrapy==1.5.0 -APScheduler==3.5.1 -fonttools==3.34.2 -elasticsearch==5.5.3 -requests==2.22.0 -pymysql==0.9.3 \ No newline at end of file diff --git a/jenkins/develop/docker-compose.yaml b/jenkins/develop/docker-compose.yaml deleted file mode 100644 index 745c0bdc..00000000 --- a/jenkins/develop/docker-compose.yaml +++ /dev/null @@ -1,32 +0,0 @@ -version: '3.3' -services: - master: - image: "tikazyq/crawlab:develop" - environment: - CRAWLAB_API_ADDRESS: "crawlab.cn/dev/api" - CRAWLAB_BASE_URL: "/dev" - CRAWLAB_SERVER_MASTER: "Y" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - CRAWLAB_LOG_PATH: "/var/logs/crawlab" - ports: - - "8090:8080" # frontend - - "8010:8000" # backend - depends_on: - - mongo - - redis - worker: - image: "tikazyq/crawlab:develop" - environment: - CRAWLAB_SERVER_MASTER: "N" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - depends_on: - - mongo - - redis - mongo: - image: mongo:latest - restart: always - redis: - image: redis:latest - restart: always diff --git a/jenkins/master/docker-compose.yaml b/jenkins/master/docker-compose.yaml deleted file mode 100644 index 139fbb39..00000000 --- a/jenkins/master/docker-compose.yaml +++ /dev/null @@ -1,51 +0,0 @@ -version: '3.3' -services: - master: - image: "tikazyq/crawlab:master" - environment: - CRAWLAB_API_ADDRESS: "https://crawlab.cn/api" - CRAWLAB_BASE_URL: "/demo" - CRAWLAB_SERVER_MASTER: "Y" - CRAWLAB_SERVER_REGISTER_TYPE: "ip" - CRAWLAB_SERVER_REGISTER_IP: "172.19.0.1" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - CRAWLAB_LOG_PATH: "/var/logs/crawlab" - CRAWLAB_SETTING_ALLOWREGISTER: "Y" - CRAWLAB_SERVER_LANG_NODE: "Y" - ports: - - "8080:8080" # frontend - depends_on: - - mongo - - redis - volumes: - - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 - worker: - image: "tikazyq/crawlab:master" - environment: - CRAWLAB_SERVER_MASTER: "N" - CRAWLAB_SERVER_REGISTER_TYPE: "ip" - CRAWLAB_SERVER_REGISTER_IP: "172.19.0.2" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - CRAWLAB_SERVER_LANG_NODE: "Y" - depends_on: - - mongo - - redis - volumes: - - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 - mongo: - image: mongo:latest - restart: always - volumes: - - "/opt/crawlab/mongo/data/db:/data/db" - - "/opt/crawlab/mongo/tmp:/tmp" - redis: - image: redis:latest - restart: always - volumes: - - "/opt/crawlab/redis/data:/data" - splash: # use Splash to run spiders on dynamic pages - image: scrapinghub/splash - # ports: - # - "8050:8050" From a34017cac12c71895f77363490e02d3da9ab9d1b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 12:00:53 +0800 Subject: [PATCH 04/15] updated devops files --- .github/workflows/dockerpush.yml | 4 +++ devops/develop/crawlab-master.yaml | 49 ++++++++++++++++++++++++++++++ devops/develop/crawlab-worker.yaml | 32 +++++++++++++++++++ devops/develop/mongo-pv.yaml | 28 +++++++++++++++++ devops/develop/mongo.yaml | 41 +++++++++++++++++++++++++ devops/develop/ns.yaml | 4 +++ devops/develop/redis.yaml | 34 +++++++++++++++++++++ devops/release/crawlab-master.yaml | 4 +++ devops/release/crawlab-worker.yaml | 6 +++- k8s/crawlab-master.yaml | 4 +++ k8s/crawlab-worker.yaml | 4 +++ 11 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 devops/develop/crawlab-master.yaml create mode 100644 devops/develop/crawlab-worker.yaml create mode 100644 devops/develop/mongo-pv.yaml create mode 100644 devops/develop/mongo.yaml create mode 100644 devops/develop/ns.yaml create mode 100644 devops/develop/redis.yaml diff --git a/.github/workflows/dockerpush.yml b/.github/workflows/dockerpush.yml index 61be3f8a..be968fe2 100644 --- a/.github/workflows/dockerpush.yml +++ b/.github/workflows/dockerpush.yml @@ -6,6 +6,7 @@ on: branches: - master - release + - develop # Publish `v1.2.3` tags as releases. tags: @@ -60,3 +61,6 @@ jobs: if [ "$VERSION" == "release" ]; then curl ${{ secrets.JENKINS_RELEASE_URL }} fi + if [ "$VERSION" == "develop" ]; then + curl ${{ secrets.JENKINS_DEVELOP_URL }} + fi diff --git a/devops/develop/crawlab-master.yaml b/devops/develop/crawlab-master.yaml new file mode 100644 index 00000000..68015e30 --- /dev/null +++ b/devops/develop/crawlab-master.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: crawlab + namespace: crawlab-develop +spec: + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30108 + selector: + app: crawlab-master + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-master + namespace: crawlab-develop +spec: + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-master + template: + metadata: + labels: + app: crawlab-master + spec: + containers: + - image: tikazyq/crawlab:develop + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "Y" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + - name: CRAWLAB_SETTING_ALLOWREGISTER + value: "Y" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" + ports: + - containerPort: 8080 + name: crawlab \ No newline at end of file diff --git a/devops/develop/crawlab-worker.yaml b/devops/develop/crawlab-worker.yaml new file mode 100644 index 00000000..7522a581 --- /dev/null +++ b/devops/develop/crawlab-worker.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawlab-worker + namespace: crawlab-develop +spec: + replicas: 2 + strategy: + type: Recreate + selector: + matchLabels: + app: crawlab-worker + template: + metadata: + labels: + app: crawlab-worker + spec: + containers: + - image: tikazyq/crawlab:develop + name: crawlab + env: + - name: CRAWLAB_SERVER_MASTER + value: "N" + - name: CRAWLAB_MONGO_HOST + value: "mongo" + - name: CRAWLAB_REDIS_ADDRESS + value: "redis" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" + \ No newline at end of file diff --git a/devops/develop/mongo-pv.yaml b/devops/develop/mongo-pv.yaml new file mode 100644 index 00000000..e47a0ed9 --- /dev/null +++ b/devops/develop/mongo-pv.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: mongo-pv-volume-develop + namespace: crawlab-develop + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data/crawlab-develop/mongodb/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-pv-claim-develop + namespace: crawlab-develop +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi \ No newline at end of file diff --git a/devops/develop/mongo.yaml b/devops/develop/mongo.yaml new file mode 100644 index 00000000..21a685bb --- /dev/null +++ b/devops/develop/mongo.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongo + namespace: crawlab-develop +spec: + ports: + - port: 27017 + selector: + app: mongo + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: mongo + namespace: crawlab-develop +spec: + selector: + matchLabels: + app: mongo + strategy: + type: Recreate + template: + metadata: + labels: + app: mongo + spec: + containers: + - image: mongo:4 + name: mongo + ports: + - containerPort: 27017 + name: mongo + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + volumes: + - name: mongo-persistent-storage + persistentVolumeClaim: + claimName: mongo-pv-claim-develop \ No newline at end of file diff --git a/devops/develop/ns.yaml b/devops/develop/ns.yaml new file mode 100644 index 00000000..bef4391e --- /dev/null +++ b/devops/develop/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: crawlab-develop \ No newline at end of file diff --git a/devops/develop/redis.yaml b/devops/develop/redis.yaml new file mode 100644 index 00000000..204e3242 --- /dev/null +++ b/devops/develop/redis.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: crawlab-develop +spec: + ports: + - port: 6379 + selector: + app: redis + clusterIP: None +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: redis + namespace: crawlab-develop +spec: + selector: + matchLabels: + app: redis + strategy: + type: Recreate + template: + metadata: + labels: + app: redis + spec: + containers: + - image: redis + name: redis + ports: + - containerPort: 6379 + name: redis \ No newline at end of file diff --git a/devops/release/crawlab-master.yaml b/devops/release/crawlab-master.yaml index e886fdd3..759ef6b4 100644 --- a/devops/release/crawlab-master.yaml +++ b/devops/release/crawlab-master.yaml @@ -40,6 +40,10 @@ spec: value: "redis" - name: CRAWLAB_SETTING_ALLOWREGISTER value: "Y" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" ports: - containerPort: 8080 name: crawlab \ No newline at end of file diff --git a/devops/release/crawlab-worker.yaml b/devops/release/crawlab-worker.yaml index 5bc2f396..986ee0c0 100644 --- a/devops/release/crawlab-worker.yaml +++ b/devops/release/crawlab-worker.yaml @@ -24,4 +24,8 @@ spec: - name: CRAWLAB_MONGO_HOST value: "mongo" - name: CRAWLAB_REDIS_ADDRESS - value: "redis" \ No newline at end of file + value: "redis" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" diff --git a/k8s/crawlab-master.yaml b/k8s/crawlab-master.yaml index b87f6881..62ac6bca 100644 --- a/k8s/crawlab-master.yaml +++ b/k8s/crawlab-master.yaml @@ -38,6 +38,10 @@ spec: value: "mongo" - name: CRAWLAB_REDIS_ADDRESS value: "redis" + # - name: CRAWLAB_SERVER_LANG_NODE + # value: "Y" + # - name: CRAWLAB_SERVER_LANG_JAVA + # value: "Y" ports: - containerPort: 8080 name: crawlab diff --git a/k8s/crawlab-worker.yaml b/k8s/crawlab-worker.yaml index b54a7796..35ae200a 100644 --- a/k8s/crawlab-worker.yaml +++ b/k8s/crawlab-worker.yaml @@ -25,3 +25,7 @@ spec: value: "mongo" - name: CRAWLAB_REDIS_ADDRESS value: "redis" + # - name: CRAWLAB_SERVER_LANG_NODE + # value: "Y" + # - name: CRAWLAB_SERVER_LANG_JAVA + # value: "Y" From 42a2eac3f5e622c83475edbc8be261ad0ff50045 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 12:23:03 +0800 Subject: [PATCH 05/15] updated install-java.sh --- backend/scripts/install-java.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh index e98de224..d01d0247 100644 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -4,6 +4,7 @@ touch /tmp/install-java.lock # install java +apt-get update apt-get install -y default-jdk ln -s /usr/bin/java /usr/local/bin/java From eafb2c70c8770ea84c6ae40b6a63cae949727da2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 14:29:42 +0800 Subject: [PATCH 06/15] code cleanup --- Dockerfile | 4 +++- Dockerfile.local | 4 +++- crawlab.conf | 5 ----- {frontend/conf => nginx}/crawlab.conf | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 crawlab.conf rename {frontend/conf => nginx}/crawlab.conf (94%) diff --git a/Dockerfile b/Dockerfile index d161ef1e..48f41d5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,9 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server # copy frontend files COPY --from=frontend-build /app/dist /app/dist -COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d + +# copy nginx config files +COPY ./nginx/crawlab.conf /etc/nginx/conf.d # working directory WORKDIR /app/backend diff --git a/Dockerfile.local b/Dockerfile.local index eb06b7e6..d49df7db 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -49,7 +49,9 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server # copy frontend files COPY --from=frontend-build /app/dist /app/dist -COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d + +# copy nginx config files +COPY ./nginx/crawlab.conf /etc/nginx/conf.d # working directory WORKDIR /app/backend diff --git a/crawlab.conf b/crawlab.conf deleted file mode 100644 index f0b7cef2..00000000 --- a/crawlab.conf +++ /dev/null @@ -1,5 +0,0 @@ -server { - listen 8080; - root /opt/crawlab/frontend/dist; - index index.html; -} \ No newline at end of file diff --git a/frontend/conf/crawlab.conf b/nginx/crawlab.conf similarity index 94% rename from frontend/conf/crawlab.conf rename to nginx/crawlab.conf index 80282fc4..fe911819 100644 --- a/frontend/conf/crawlab.conf +++ b/nginx/crawlab.conf @@ -2,7 +2,6 @@ server { gzip on; gzip_min_length 1k; gzip_buffers 4 16k; - #gzip_http_version 1.0; gzip_comp_level 2; gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; gzip_vary off; From e6826b4de165925ee73f3bbedf1e36b24a254162 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 14:33:41 +0800 Subject: [PATCH 07/15] fixed https://github.com/crawlab-team/crawlab/issues/616 --- nginx/crawlab.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/nginx/crawlab.conf b/nginx/crawlab.conf index fe911819..36c9f2b1 100644 --- a/nginx/crawlab.conf +++ b/nginx/crawlab.conf @@ -13,5 +13,6 @@ server { location /api/ { rewrite /api/(.*) /$1 break; proxy_pass http://localhost:8000/; + client_max_body_size 50m; } } From cef919a2dec89315fb540e0cc0acc4f781e3c9a2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 14:48:03 +0800 Subject: [PATCH 08/15] updated imagePullPolicy --- devops/develop/crawlab-master.yaml | 1 + devops/develop/crawlab-worker.yaml | 1 + devops/release/crawlab-master.yaml | 1 + devops/release/crawlab-worker.yaml | 1 + k8s/crawlab-master.yaml | 1 + k8s/crawlab-worker.yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/devops/develop/crawlab-master.yaml b/devops/develop/crawlab-master.yaml index 68015e30..1168c404 100644 --- a/devops/develop/crawlab-master.yaml +++ b/devops/develop/crawlab-master.yaml @@ -30,6 +30,7 @@ spec: spec: containers: - image: tikazyq/crawlab:develop + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/devops/develop/crawlab-worker.yaml b/devops/develop/crawlab-worker.yaml index 7522a581..096f0ccc 100644 --- a/devops/develop/crawlab-worker.yaml +++ b/devops/develop/crawlab-worker.yaml @@ -17,6 +17,7 @@ spec: spec: containers: - image: tikazyq/crawlab:develop + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/devops/release/crawlab-master.yaml b/devops/release/crawlab-master.yaml index 759ef6b4..342d1300 100644 --- a/devops/release/crawlab-master.yaml +++ b/devops/release/crawlab-master.yaml @@ -30,6 +30,7 @@ spec: spec: containers: - image: tikazyq/crawlab:release + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/devops/release/crawlab-worker.yaml b/devops/release/crawlab-worker.yaml index 986ee0c0..bf96607e 100644 --- a/devops/release/crawlab-worker.yaml +++ b/devops/release/crawlab-worker.yaml @@ -17,6 +17,7 @@ spec: spec: containers: - image: tikazyq/crawlab:release + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/k8s/crawlab-master.yaml b/k8s/crawlab-master.yaml index 62ac6bca..758cc051 100644 --- a/k8s/crawlab-master.yaml +++ b/k8s/crawlab-master.yaml @@ -30,6 +30,7 @@ spec: spec: containers: - image: tikazyq/crawlab:latest + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER diff --git a/k8s/crawlab-worker.yaml b/k8s/crawlab-worker.yaml index 35ae200a..2efcd556 100644 --- a/k8s/crawlab-worker.yaml +++ b/k8s/crawlab-worker.yaml @@ -17,6 +17,7 @@ spec: spec: containers: - image: tikazyq/crawlab:latest + imagePullPolicy: Always name: crawlab env: - name: CRAWLAB_SERVER_MASTER From fadc62fcc86028284a0491ffbf6bd2328002a3c2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 4 Mar 2020 15:09:15 +0800 Subject: [PATCH 09/15] updated crawlab.conf --- nginx/crawlab.conf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nginx/crawlab.conf b/nginx/crawlab.conf index 36c9f2b1..d04da86c 100644 --- a/nginx/crawlab.conf +++ b/nginx/crawlab.conf @@ -6,13 +6,13 @@ server { gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; gzip_vary off; gzip_disable "MSIE [1-6]\."; - listen 8080; - root /app/dist; - index index.html; + client_max_body_size 50m; + listen 8080; + root /app/dist; + index index.html; - location /api/ { - rewrite /api/(.*) /$1 break; - proxy_pass http://localhost:8000/; - client_max_body_size 50m; - } + location /api/ { + rewrite /api/(.*) /$1 break; + proxy_pass http://localhost:8000/; + } } From e24f4fced7b1c32581b9e3d52b16c8df1194315b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 07:48:48 +0800 Subject: [PATCH 10/15] updated logs --- docker_init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker_init.sh b/docker_init.sh index 9c51fa83..19cf8471 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -26,14 +26,14 @@ service nginx start if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; then echo "installing node.js" - /bin/sh /app/backend/scripts/install-nodejs.sh & + /bin/sh /app/backend/scripts/install-nodejs.sh >> /var/log/install-nodejs.sh.log 2>&1 & fi # install languages: Java if [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ]; then echo "installing java" - /bin/sh /app/backend/scripts/install-java.sh & + /bin/sh /app/backend/scripts/install-java.sh >> /var/log/install-java.sh.log 2>&1 & fi # generate ssh From 9e24d22ee1d876ccf2ad66c558e9cb9e620ace46 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 08:09:12 +0800 Subject: [PATCH 11/15] updated install-java.sh --- backend/constants/register.go | 7 +++++++ backend/model/node.go | 1 + backend/scripts/install-java.sh | 3 +-- backend/services/node.go | 8 ++++++++ backend/services/register/register.go | 6 ++++-- 5 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 backend/constants/register.go mode change 100644 => 100755 backend/scripts/install-java.sh diff --git a/backend/constants/register.go b/backend/constants/register.go new file mode 100644 index 00000000..ad38e7a3 --- /dev/null +++ b/backend/constants/register.go @@ -0,0 +1,7 @@ +package constants + +const ( + RegisterTypeMac = "mac" + RegisterTypeIp = "ip" + RegisterTypeHostname = "hostname" +) diff --git a/backend/model/node.go b/backend/model/node.go index 88c4ed66..51cd5d7e 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -20,6 +20,7 @@ type Node struct { Ip string `json:"ip" bson:"ip"` Port string `json:"port" bson:"port"` Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` Description string `json:"description" bson:"description"` // 用于唯一标识节点，可能是mac地址，可能是ip地址 Key string `json:"key" bson:"key"` diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh old mode 100644 new mode 100755 index d01d0247..c42a2901 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -4,8 +4,7 @@ touch /tmp/install-java.lock # install java -apt-get update -apt-get install -y default-jdk +apt-get update && apt-get install -y default-jdk ln -s /usr/bin/java /usr/local/bin/java # unlock diff --git a/backend/services/node.go b/backend/services/node.go index d6124205..f189ddb3 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -161,6 +161,14 @@ func UpdateNodeData() { log.Errorf(err.Error()) return } + + // 获取Hostname + hostname, err := register.GetRegister().GetHostname() + if err != nil { + log.Errorf(err.Error()) + return + } + // 获取redis的key key, err := register.GetRegister().GetKey() if err != nil { diff --git a/backend/services/register/register.go b/backend/services/register/register.go index ed4e1891..bbdec2f3 100644 --- a/backend/services/register/register.go +++ b/backend/services/register/register.go @@ -1,6 +1,7 @@ package register import ( + "crawlab/constants" "github.com/apex/log" "github.com/spf13/viper" "net" @@ -108,9 +109,9 @@ func GetRegister() Register { } registerType := viper.GetString("server.register.type") - if registerType == "mac" { + if registerType == constants.RegisterTypeMac { register = &MacRegister{} - } else { + } else if registerType == constants.RegisterTypeIp { ip := viper.GetString("server.register.ip") if ip == "" { log.Error("server.register.ip is empty") @@ -120,6 +121,7 @@ func GetRegister() Register { register = &IpRegister{ Ip: ip, } + } else if registerType == constants.RegisterTypeHostname { } log.Info("register type is :" + reflect.TypeOf(register).String()) From a3326da0f540376cc4208bc9e2e9cb8c1f56d484 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 08:56:01 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=E5=8A=A0=E5=85=A5hostname=E6=B3=A8?= =?UTF-8?q?=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 2 +- backend/model/node.go | 20 +++++++--- backend/services/node.go | 3 +- backend/services/register/register.go | 54 +++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 32fce297..1e98a2eb 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -23,7 +23,7 @@ server: master: "Y" secret: "crawlab" register: - # mac地址 或者 ip地址，如果是ip，则需要手动指定IP + # mac地址/ip地址/hostname, 如果是ip，则需要手动指定IP type: "mac" ip: "" lang: # 安装语言环境, Y 为安装，N 为不安装，只对 Docker 有效 diff --git a/backend/model/node.go b/backend/model/node.go index 51cd5d7e..6e29a073 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -43,6 +43,7 @@ func IsMaster() bool { } // 获取本机节点 +// TODO: 这里职责不单一，需要重构 func GetCurrentNode() (Node, error) { // 获得注册的key值 key, err := register.GetRegister().GetKey() @@ -68,7 +69,7 @@ func GetCurrentNode() (Node, error) { //只在master节点运行的时候才检测master节点的信息是否存在 if IsMaster() && err == mgo.ErrNotFound { // 获取本机信息 - ip, mac, key, err := GetNodeBaseInfo() + ip, mac, hostname, key, err := GetNodeBaseInfo() if err != nil { debug.PrintStack() return node, err @@ -81,6 +82,7 @@ func GetCurrentNode() (Node, error) { Ip: ip, Name: ip, Mac: mac, + Hostname: hostname, IsMaster: true, } if err := node.Add(); err != nil { @@ -240,25 +242,31 @@ func GetNodeCount(query interface{}) (int, error) { } // 节点基本信息 -func GetNodeBaseInfo() (ip string, mac string, key string, error error) { +func GetNodeBaseInfo() (ip string, mac string, hostname string, key string, error error) { ip, err := register.GetRegister().GetIp() if err != nil { debug.PrintStack() - return "", "", "", err + return "", "", "", "", err } mac, err = register.GetRegister().GetMac() if err != nil { debug.PrintStack() - return "", "", "", err + return "", "", "", "", err + } + + hostname, err = register.GetRegister().GetHostname() + if err != nil { + debug.PrintStack() + return "", "", "", "", err } key, err = register.GetRegister().GetKey() if err != nil { debug.PrintStack() - return "", "", "", err + return "", "", "", "", err } - return ip, mac, key, nil + return ip, mac, key, hostname, nil } // 根据redis的key值，重置node节点为offline diff --git a/backend/services/node.go b/backend/services/node.go index f189ddb3..f8db160a 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -23,6 +23,7 @@ type Data struct { Key string `json:"key"` Mac string `json:"mac"` Ip string `json:"ip"` + Hostname string `json:"hostname"` Master bool `json:"master"` UpdateTs time.Time `json:"update_ts"` UpdateTsUnix int64 `json:"update_ts_unix"` @@ -182,6 +183,7 @@ func UpdateNodeData() { Key: key, Mac: mac, Ip: ip, + Hostname: hostname, Master: model.IsMaster(), UpdateTs: time.Now(), UpdateTsUnix: time.Now().Unix(), @@ -199,7 +201,6 @@ func UpdateNodeData() { log.Errorf(err.Error()) return } - } func MasterNodeCallback(message redis.Message) (err error) { diff --git a/backend/services/register/register.go b/backend/services/register/register.go index bbdec2f3..8169ae65 100644 --- a/backend/services/register/register.go +++ b/backend/services/register/register.go @@ -1,10 +1,13 @@ package register import ( + "bytes" "crawlab/constants" + "fmt" "github.com/apex/log" "github.com/spf13/viper" "net" + "os/exec" "reflect" "runtime/debug" "sync" @@ -19,6 +22,8 @@ type Register interface { GetIp() (string, error) // 注册节点的mac地址 GetMac() (string, error) + // 注册节点的Hostname + GetHostname() (string, error) } // ===================== mac 地址注册 ===================== @@ -40,6 +45,10 @@ func (mac *MacRegister) GetIp() (string, error) { return getIp() } +func (mac *MacRegister) GetHostname() (string, error) { + return getHostname() +} + // ===================== ip 地址注册 ===================== type IpRegister struct { Ip string @@ -61,6 +70,33 @@ func (ip *IpRegister) GetMac() (string, error) { return getMac() } +func (ip *IpRegister) GetHostname() (string, error) { + return getHostname() +} + +// ===================== mac 地址注册 ===================== +type HostnameRegister struct{} + +func (h *HostnameRegister) GetType() string { + return "mac" +} + +func (h *HostnameRegister) GetKey() (string, error) { + return h.GetHostname() +} + +func (h *HostnameRegister) GetMac() (string, error) { + return getMac() +} + +func (h *HostnameRegister) GetIp() (string, error) { + return getIp() +} + +func (h *HostnameRegister) GetHostname() (string, error) { + return getHostname() +} + // ===================== 公共方法 ===================== // 获取本机的IP地址 // TODO: 考虑多个IP地址的情况 @@ -95,6 +131,23 @@ func getMac() (string, error) { return "", nil } +func getHostname() (string, error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("hostname") + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf(fmt.Sprintf("error: %s", stderr.String())) + debug.PrintStack() + return "", err + } + + return stdout.String(), nil +} + // ===================== 获得注册简单工厂 ===================== var register Register @@ -122,6 +175,7 @@ func GetRegister() Register { Ip: ip, } } else if registerType == constants.RegisterTypeHostname { + register = &HostnameRegister{} } log.Info("register type is :" + reflect.TypeOf(register).String()) From 1409cfb0f8b5ddcb53412e0a97e8628bbe69a5cd Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 09:31:11 +0800 Subject: [PATCH 13/15] try to fix install java error --- backend/scripts/install-java.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh index c42a2901..d99b16be 100755 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -4,7 +4,7 @@ touch /tmp/install-java.lock # install java -apt-get update && apt-get install -y default-jdk +apt-get update && apt-get install -y default-jdk --fix-missing ln -s /usr/bin/java /usr/local/bin/java # unlock From 13963528cada08e0922b04f71a70f72cb9e5645f Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 11:01:26 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/utils/tour.js | 1 + frontend/src/views/node/NodeDetail.vue | 1 - frontend/src/views/schedule/ScheduleList.vue | 2 -- frontend/src/views/setting/Setting.vue | 1 - frontend/src/views/spider/SpiderDetail.vue | 2 -- frontend/src/views/spider/SpiderList.vue | 2 -- frontend/src/views/task/TaskDetail.vue | 1 - frontend/src/views/task/TaskList.vue | 1 - 8 files changed, 1 insertion(+), 10 deletions(-) diff --git a/frontend/src/utils/tour.js b/frontend/src/utils/tour.js index 33945603..7696c8ed 100644 --- a/frontend/src/utils/tour.js +++ b/frontend/src/utils/tour.js @@ -21,6 +21,7 @@ export default { startTour: (vm, tourName) => { if (localStorage.getItem('enableTutorial') === '0') return vm.$tours[tourName].start() + vm.$st.sendEv('教程', '开始', tourName) }, finishTour: (tourName) => { let data diff --git a/frontend/src/views/node/NodeDetail.vue b/frontend/src/views/node/NodeDetail.vue index 7f87a87e..bfef601b 100644 --- a/frontend/src/views/node/NodeDetail.vue +++ b/frontend/src/views/node/NodeDetail.vue @@ -125,7 +125,6 @@ export default { mounted () { if (!this.$utils.tour.isFinishedTour('node-detail')) { this.$utils.tour.startTour(this, 'node-detail') - this.$st.sendEv('教程', '开始', 'node-detail') } } } diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index adbfef56..68f8b012 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -468,7 +468,6 @@ export default { if (!this.$utils.tour.isFinishedTour('schedule-list-add')) { setTimeout(() => { this.$utils.tour.startTour(this, 'schedule-list-add') - this.$st.sendEv('教程', '开始', 'schedule-list-add') }, 500) } }, @@ -618,7 +617,6 @@ export default { if (!this.isDisabledSpiderSchedule) { if (!this.$utils.tour.isFinishedTour('schedule-list')) { this.$utils.tour.startTour(this, 'schedule-list') - this.$st.sendEv('教程', '开始', 'schedule-list') } } } diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue index 86118e71..c2c7c7f2 100644 --- a/frontend/src/views/setting/Setting.vue +++ b/frontend/src/views/setting/Setting.vue @@ -315,7 +315,6 @@ export default { mounted () { if (!this.$utils.tour.isFinishedTour('setting')) { this.$utils.tour.startTour(this, 'setting') - this.$st.sendEv('教程', '开始', 'setting') } } } diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index a69545b2..02038caf 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -209,7 +209,6 @@ export default { if (!this.$utils.tour.isFinishedTour('spider-detail-config')) { setTimeout(() => { this.$utils.tour.startTour(this, 'spider-detail-config') - this.$st.sendEv('教程', '开始', 'spider-detail-config') }, 100) } } else if (this.activeTabName === 'scrapy-settings') { @@ -261,7 +260,6 @@ export default { mounted () { if (!this.$utils.tour.isFinishedTour('spider-detail')) { this.$utils.tour.startTour(this, 'spider-detail') - this.$st.sendEv('教程', '开始', 'spider-detail') } } } diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 8e6a6ba9..10dca8c0 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -888,7 +888,6 @@ export default { setTimeout(() => { if (!this.$utils.tour.isFinishedTour('spider-list-add')) { this.$utils.tour.startTour(this, 'spider-list-add') - this.$st.sendEv('教程', '开始', 'spider-list-add') } }, 300) }, @@ -1242,7 +1241,6 @@ export default { if (!this.$utils.tour.isFinishedTour('spider-list')) { this.$utils.tour.startTour(this, 'spider-list') - this.$st.sendEv('教程', '开始', 'spider-list') } }, destroyed () { diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index 422dc088..0ecbdf9e 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -203,7 +203,6 @@ export default { mounted () { if (!this.$utils.tour.isFinishedTour('task-detail')) { this.$utils.tour.startTour(this, 'task-detail') - this.$st.sendEv('教程', '开始', 'task-detail') } }, destroyed () { diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index 3cd564d9..1425e2da 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -413,7 +413,6 @@ export default { if (!this.$utils.tour.isFinishedTour('task-list')) { this.$utils.tour.startTour(this, 'task-list') - this.$st.sendEv('教程', '开始', 'task-list') } }, destroyed () { From f0382ab7680bfd19c4b46e3a416fe0d7f4fe4cc4 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 5 Mar 2020 11:01:58 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=E6=9A=82=E6=97=B6=E5=85=B3=E9=97=ADjava?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/system.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/system.go b/backend/services/system.go index b2884378..285cad5a 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -67,7 +67,7 @@ func GetLangList(nodeId string) []entity.Lang { list := []entity.Lang{ {Name: "Python", ExecutableName: "python", ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, DepExecutablePath: "/usr/local/bin/pip"}, {Name: "Node.js", ExecutableName: "node", ExecutablePaths: []string{"/usr/bin/node", "/usr/local/bin/node"}, DepExecutablePath: "/usr/local/bin/npm"}, - {Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}}, + //{Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}}, } for i, lang := range list { list[i].Installed = IsInstalledLang(nodeId, lang)