added air for debugging containers

2026-01-21 17:21:09 +01:00 · 2021-11-20 11:30:55 +08:00
parent 3b5119e055
commit fa4dc4b0f8
7 changed files with 132 additions and 88 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ tmp/
 _book/
 *.lock
 vendor/
+.crawlab
--- a/backend/.air.master.conf
+++ b/backend/.air.master.conf
@@ -0,0 +1,47 @@
+# Config file for [Air](https://github.com/cosmtrek/air) in TOML format
+
+# Working directory
+# . or absolute path, please note that the directories following must be under root.
+root = "."
+tmp_dir = "/tmp"
+
+[build]
+# Just plain old shell command. You could use `make` as well.
+cmd = "go build  -o ../tmp/main ./ "
+# Binary file yields from `cmd`.
+bin = "../tmp/main"
+# Customize binary.
+full_bin = "../tmp/main master"
+# Watch these filename extensions.
+include_ext = ["go", "tpl", "tmpl", "html"]
+# Ignore these filename extensions or directories.
+exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
+# Watch these directories if you specified.
+include_dir = ["../libs"]
+# Exclude files.
+exclude_file = [
+# This log file places in your tmp_dir.
+log = "air.log"
+# It's not necessary to trigger build each time file changes if it's too frequent.
+delay = 1000 # ms
+# Stop running old binary when build errors occur.
+stop_on_error = true
+# Send Interrupt signal before killing process (windows does not support this feature)
+send_interrupt = false
+# Delay after sending Interrupt signal
+kill_delay = 500 # ms
+
+[log]
+# Show log time
+time = false
+
+[color]
+# Customize each part's color. If no color found, use the raw app log.
+main = "magenta"
+watcher = "cyan"
+build = "yellow"
+runner = "green"
+
+[misc]
+# Delete tmp directory on exit
+clean_on_exit = true
--- a/backend/.air.worker.conf
+++ b/backend/.air.worker.conf
@@ -11,13 +11,13 @@ cmd = "go build  -o ../tmp/main ./ "
 # Binary file yields from `cmd`.
 bin = "../tmp/main"
 # Customize binary.
-full_bin = "../tmp/main start"
+full_bin = "../tmp/main worker"
 # Watch these filename extensions.
 include_ext = ["go", "tpl", "tmpl", "html"]
 # Ignore these filename extensions or directories.
 exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
 # Watch these directories if you specified.
-include_dir = []
+include_dir = ["../libs"]
 # Exclude files.
 exclude_file = []
 # This log file places in your tmp_dir.
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -1,6 +1,6 @@
 module crawlab

-go 1.15
+go 1.16

 require (
 	github.com/apex/log v1.9.0
--- a/backend/go.mod.local
+++ b/backend/go.mod.local
@@ -0,0 +1,21 @@
+module crawlab
+
+go 1.15
+
+replace (
+    github.com/crawlab-team/crawlab-core => /libs/crawlab-team/crawlab-core
+    github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs
+    github.com/crawlab-team/crawlab-fs => /libs/crawlab-team/crawlab-fs
+    github.com/crawlab-team/crawlab-db => /libs/crawlab-team/crawlab-db
+)
+
+require (
+	github.com/apex/log v1.9.0
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050
+	github.com/crawlab-team/go-trace v0.1.0
+	github.com/gin-gonic/gin v1.6.3
+	github.com/spf13/cobra v1.1.3
+	github.com/spf13/viper v1.7.1
+	go.mongodb.org/mongo-driver v1.6.0 // indirect
+	go.uber.org/dig v1.10.0
+)
--- a/workspace/docker-compose.yml
+++ b/workspace/docker-compose.yml
@@ -1,95 +1,56 @@
-version: "3.3"
+version: '3.3'
 services:
  master:
    build:
-      context: dockerfiles/golang
-    command: "air -c .air.conf"
+      context: ./dockerfiles/golang
+    command: "air -c .air.master.conf"
    volumes:
+      - "./.crawlab/master:/root/.crawlab"
      - ../backend:/backend
-      - /backend/tmp
-    depends_on:
-      - mongo
-      - redis
-    ports:
-      - 8000:8000
+      - ../backend/go.mod.local:/backend/go.mod
+      - ../..:/libs/crawlab-team
    environment:
-      CRAWLAB_SPIDER_PATH: "/spiders"
-      CRAWLAB_SETTING_ENABLEDEMOSPIDERS: "Y"
+      CRAWLAB_NODE_MASTER: "Y"
      CRAWLAB_MONGO_HOST: "mongo"
-      CRAWLAB_REDIS_ADDRESS: "redis"
-      CRAWLAB_SERVER_MASTER: "Y"
-      CRAWLAB_SERVER_REGISTER_TYPE: "customName"
-      CRAWLAB_SERVER_REGISTER_CUSTOMNODENAME: "master_1"
-      CRAWLAB_SERVER_PORT: 8000
-  worker-1:
-    build:
-      context: dockerfiles/golang
-    command: "air -c .air.conf"
+    ports:
+      - "9080:8080"
    depends_on:
      - mongo
-      - redis
-    ports:
-      - 8001:8000
-    volumes:
-      - ../backend:/backend
-      - /backend/tmp

-    environment:
-      CRAWLAB_SPIDER_PATH: "/spiders"
-      CRAWLAB_MONGO_HOST: "mongo"
-      CRAWLAB_REDIS_ADDRESS: "redis"
-      CRAWLAB_SERVER_MASTER: "N"
-      CRAWLAB_SERVER_PORT: 8001
-      CRAWLAB_SERVER_REGISTER_TYPE: "customName"
-      CRAWLAB_SERVER_REGISTER_CUSTOMNODENAME: "worker_1"
-  worker-2:
+  worker01:
    build:
-      context: dockerfiles/golang
-    command: "air -c .air.conf"
-    depends_on:
-      - mongo
-      - redis
-    ports:
-      - 8002:8000
+      context: ./dockerfiles/golang
+    command: "air -c .air.worker.conf"
+    environment:
+      CRAWLAB_NODE_MASTER: "N"
+      CRAWLAB_GRPC_ADDRESS: "master"
+      CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
    volumes:
+      - "./.crawlab/worker01:/root/.crawlab"
      - ../backend:/backend
-    environment:
-      CRAWLAB_SPIDER_PATH: "/spiders"
-      CRAWLAB_MONGO_HOST: "mongo"
-      CRAWLAB_REDIS_ADDRESS: "redis"
-      CRAWLAB_SERVER_MASTER: "N"
-      CRAWLAB_SERVER_PORT: 8002
-      CRAWLAB_SERVER_REGISTER_TYPE: "customName"
-      CRAWLAB_SERVER_REGISTER_CUSTOMNODENAME: "worker_2"
-  ui:
-    build:
-      context: ./dockerfiles/node
-    container_name: crawlab_frontend
-    ports:
-      - 8080:8080
-    volumes:
-      - ../frontend:/frontend
-      - /frontend/node_modules
-    command:
-      - /bin/sh
-      - -c
-      - |
-        yarn install
-        yarn run serve
+      - ../backend/go.mod.local:/backend/go.mod
+      - ../..:/libs/crawlab-team
    depends_on:
      - master
+
+  worker02:
+    build:
+      context: ./dockerfiles/golang
+    command: "air -c .air.worker.conf"
+    environment:
+      CRAWLAB_NODE_MASTER: "N"
+      CRAWLAB_GRPC_ADDRESS: "master"
+      CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
+    volumes:
+      - "./.crawlab/worker02:/root/.crawlab"
+      - ../backend:/backend
+      - ../backend/go.mod.local:/backend/go.mod
+      - ../..:/libs/crawlab-team
+    depends_on:
+      - master
+
  mongo:
-    image: mongo:latest
+    image: mongo:4
    restart: always
-    # volumes:
-    #   - "/opt/crawlab/mongo/data/db:/data/db"  # make data persistent 持久化
    ports:
-      - "27017:27017"  # expose port to host machine 暴露接口到宿主机
-  redis:
-    image: redis:latest
-    restart: always
-    # command: redis-server --requirepass "password" # set redis password 设置 Redis 密码
-    # volumes:
-    #   - "/opt/crawlab/redis/data:/data"  # make data persistent 持久化
-    ports:
-      - "6379:6379"  # expose port to host machine 暴露接口到宿主机
+      - "28017:27017"
--- a/workspace/dockerfiles/golang/Dockerfile
+++ b/workspace/dockerfiles/golang/Dockerfile
@@ -1,26 +1,40 @@
-FROM golang:buster
+FROM golang:1.16
+
 RUN go env -w GOPROXY=https://goproxy.io,https://goproxy.cn && \
    go env -w GO111MODULE="on"
+
 WORKDIR /tools
 RUN go get github.com/cosmtrek/air
+
 WORKDIR /backend
 RUN rm -rf /tools
+
 # set as non-interactive
 ENV DEBIAN_FRONTEND noninteractive

-# set CRAWLAB_IS_DOCKER
-ENV CRAWLAB_IS_DOCKER Y
 # install packages
 RUN chmod 777 /tmp \
-    && sed -i 's#http://deb.debian.org#https://mirrors.tuna.tsinghua.edu.cn#g' /etc/apt/sources.list \
 	&& apt-get update \
-	&& apt-get install -y curl net-tools iputils-ping ntp ntpdate python3 python3-pip dumb-init  \
+	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc
+
+# install python
+RUN apt-get install -y python3 python3-pip \
 	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
 	&& ln -s /usr/bin/python3 /usr/local/bin/python

-RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
+# install golang
+RUN curl -OL https://storage.googleapis.com/golang/go1.16.7.linux-amd64.tar.gz \
+	&& tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \
+	&& ln -s /usr/local/go/bin/go /usr/local/bin/go
+
+# install seaweedfs
+RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.76/linux_amd64.tar.gz \
+  && tar -zxf linux_amd64.tar.gz \
+  && cp weed /usr/local/bin
+
 # install backend
-RUN pip install scrapy pymongo bs4 requests crawlab-sdk scrapy-splash
-RUN mkdir /spiders && chmod -R 0755 /spiders
+RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
+RUN pip install crawlab-sdk==0.6.b20211024-1207
+
 VOLUME /backend
 EXPOSE 8080