From ec2a738958af673a8eb4d5dab4e3073fcb92f8ba Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 3 Jan 2020 15:45:18 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0docker=E5=AE=89=E8=A3=85node?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- Dockerfile.local | 2 +- backend/conf/config.yml | 3 + backend/services/system.go | 1 + docker-compose.yml | 3 + docker_init.sh | 9 +- jenkins/master/docker-compose.yaml | 1 + wait-for-it.sh | 178 ----------------------------- 8 files changed, 18 insertions(+), 181 deletions(-) delete mode 100755 wait-for-it.sh diff --git a/Dockerfile b/Dockerfile index cf8ab174..73883c64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,4 +59,4 @@ EXPOSE 8080 EXPOSE 8000 # start backend -CMD ["/bin/sh", "/app/docker_init.sh"] +CMD ["/bin/bash", "/app/docker_init.sh"] diff --git a/Dockerfile.local b/Dockerfile.local index d99010a4..59b8736d 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -57,4 +57,4 @@ EXPOSE 8080 EXPOSE 8000 # start backend -CMD ["/bin/sh", "/app/docker_init.sh"] +CMD ["/bin/bash", "/app/docker_init.sh"] diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 5ada78f6..c0c1256e 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -26,6 +26,9 @@ server: # mac地址 或者 ip地址，如果是ip，则需要手动指定IP type: "mac" ip: "" + lang: # 安装语言环境, Y 为安装，N 为不安装，只对 Docker 有效 + python: "Y" + node: "N" spider: path: "/app/spiders" task: diff --git a/backend/services/system.go b/backend/services/system.go index dce309d1..12b8744c 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -415,6 +415,7 @@ func UninstallPythonLocalDep(depName string) (string, error) { cmd := exec.Command("pip", "uninstall", "-y", depName) outputBytes, err := cmd.Output() if err != nil { + log.Errorf(string(outputBytes)) log.Errorf(err.Error()) debug.PrintStack() return fmt.Sprintf("error: %s", err.Error()), err diff --git a/docker-compose.yml b/docker-compose.yml index b4f36e86..4fe71d24 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,12 +8,15 @@ services: CRAWLAB_SERVER_MASTER: "Y" # whether to be master node 是否为主节点，主节点为 Y，工作节点为 N CRAWLAB_MONGO_HOST: "mongo" # MongoDB host address MongoDB 的地址，在 docker compose 网络中，直接引用服务名称 CRAWLAB_REDIS_ADDRESS: "redis" # Redis host address Redis 的地址，在 docker compose 网络中，直接引用服务名称 + CRAWLAB_SERVER_LANG_NODE: "Y" # 预安装 Node.js 语言环境 ports: - "8080:8080" # frontend port mapping 前端端口映射 - "8000:8000" # backend port mapping 后端端口映射 depends_on: - mongo - redis + volumes: + - "/Users/marvzhang/projects/crawlab-team/crawlab/docker_init.sh:/app/docker_init.sh" worker: image: tikazyq/crawlab:latest container_name: worker diff --git a/docker_init.sh b/docker_init.sh index 97c505dc..648634cd 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # replace default api path to new one if [ "${CRAWLAB_API_ADDRESS}" = "" ]; @@ -22,5 +22,12 @@ fi # start nginx service nginx start +# install languages: Node.js +if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; +then + echo "installing node.js" + /bin/sh /app/backend/scripts/install-nodejs.sh +fi + # start backend crawlab \ No newline at end of file diff --git a/jenkins/master/docker-compose.yaml b/jenkins/master/docker-compose.yaml index ff9dd64e..ed649aa8 100644 --- a/jenkins/master/docker-compose.yaml +++ b/jenkins/master/docker-compose.yaml @@ -9,6 +9,7 @@ services: CRAWLAB_MONGO_HOST: "mongo" CRAWLAB_REDIS_ADDRESS: "redis" CRAWLAB_LOG_PATH: "/var/logs/crawlab" + CRAWLAB_SETTING_ALLOWREGISTER: "Y" ports: - "8080:8080" # frontend - "8000:8000" # backend diff --git a/wait-for-it.sh b/wait-for-it.sh deleted file mode 100755 index 607a7d67..00000000 --- a/wait-for-it.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available - -WAITFORIT_cmdname=${0##*/} - -echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - else - echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" - fi - WAITFORIT_start_ts=$(date +%s) - while : - do - if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then - nc -z $WAITFORIT_HOST $WAITFORIT_PORT - WAITFORIT_result=$? - else - (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 - WAITFORIT_result=$? - fi - if [[ $WAITFORIT_result -eq 0 ]]; then - WAITFORIT_end_ts=$(date +%s) - echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" - break - fi - sleep 1 - done - return $WAITFORIT_result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $WAITFORIT_QUIET -eq 1 ]]; then - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - else - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - fi - WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT - wait $WAITFORIT_PID - WAITFORIT_RESULT=$? - if [[ $WAITFORIT_RESULT -ne 0 ]]; then - echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - fi - return $WAITFORIT_RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - WAITFORIT_hostport=(${1//:/ }) - WAITFORIT_HOST=${WAITFORIT_hostport[0]} - WAITFORIT_PORT=${WAITFORIT_hostport[1]} - shift 1 - ;; - --child) - WAITFORIT_CHILD=1 - shift 1 - ;; - -q | --quiet) - WAITFORIT_QUIET=1 - shift 1 - ;; - -s | --strict) - WAITFORIT_STRICT=1 - shift 1 - ;; - -h) - WAITFORIT_HOST="$2" - if [[ $WAITFORIT_HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - WAITFORIT_HOST="${1#*=}" - shift 1 - ;; - -p) - WAITFORIT_PORT="$2" - if [[ $WAITFORIT_PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - WAITFORIT_PORT="${1#*=}" - shift 1 - ;; - -t) - WAITFORIT_TIMEOUT="$2" - if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - WAITFORIT_TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - WAITFORIT_CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} -WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} -WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} -WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} - -# check to see if timeout is from busybox? -WAITFORIT_TIMEOUT_PATH=$(type -p timeout) -WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) -if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then - WAITFORIT_ISBUSY=1 - WAITFORIT_BUSYTIMEFLAG="-t" - -else - WAITFORIT_ISBUSY=0 - WAITFORIT_BUSYTIMEFLAG="" -fi - -if [[ $WAITFORIT_CHILD -gt 0 ]]; then - wait_for - WAITFORIT_RESULT=$? - exit $WAITFORIT_RESULT -else - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - wait_for_wrapper - WAITFORIT_RESULT=$? - else - wait_for - WAITFORIT_RESULT=$? - fi -fi - -if [[ $WAITFORIT_CLI != "" ]]; then - if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then - echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" - exit $WAITFORIT_RESULT - fi - exec "${WAITFORIT_CLI[@]}" -else - exit $WAITFORIT_RESULT -fi \ No newline at end of file