git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

updated docker dev process

Browse Source
This commit is contained in:
Marvin Zhang
2021-11-20 17:55:08 +08:00
parent 907897f288
commit 1450ed4941
10 changed files with 47 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207
# add files
COPY ./backend/conf /app/backend/conf
COPY ./nginx /app/nginx
COPY ./docker_init.sh /app/docker_init.sh
COPY ./bin /app/bin
# copy backend files
RUN mkdir -p /opt/bin
@@ -84,4 +84,4 @@ EXPOSE 8080
EXPOSE 8000
# start backend
CMD ["/bin/bash", "/app/docker_init.sh"]
CMD ["/bin/bash", "/app/bin/docker-init.sh"]

4
Dockerfile.cn
View File

@@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207
# add files
COPY ./backend/conf /app/backend/conf
COPY ./nginx /app/nginx
COPY ./docker_init.sh /app/docker_init.sh
COPY ./bin /app/bin
# copy backend files
RUN mkdir -p /opt/bin
@@ -84,4 +84,4 @@ EXPOSE 8080
EXPOSE 8000
# start backend
CMD ["/bin/bash", "/app/docker_init.sh"]
CMD ["/bin/bash", "/app/bin/docker-init.sh"]

4
Dockerfile.local
View File

@@ -54,7 +54,7 @@ RUN pip install crawlab-sdk==0.6.b20211024-1207
# add files
COPY ./backend/conf /app/backend/conf
COPY ./nginx /app/nginx
COPY ./docker_init.sh /app/docker_init.sh
COPY ./bin /app/bin
# copy backend files
RUN mkdir -p /opt/bin
@@ -84,4 +84,4 @@ EXPOSE 8080
EXPOSE 8000
# start backend
CMD ["/bin/bash", "/app/docker_init.sh"]
CMD ["/bin/bash", "/app/bin/docker-init.sh"]

4
backend/.air.master.conf
View File

@@ -7,7 +7,7 @@ tmp_dir = "/tmp"
[build]
# Just plain old shell command. You could use `make` as well.
cmd = "go build -o ../tmp/main ./ "
cmd = "go build -o ../tmp/main ./ "
# Binary file yields from `cmd`.
bin = "../tmp/main"
# Customize binary.
@@ -19,7 +19,7 @@ exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
# Watch these directories if you specified.
include_dir = ["../libs"]
# Exclude files.
exclude_file = [
exclude_file = []
# This log file places in your tmp_dir.
log = "air.log"
# It's not necessary to trigger build each time file changes if it's too frequent.

2
backend/.air.worker.conf
View File

@@ -7,7 +7,7 @@ tmp_dir = "/tmp"
[build]
# Just plain old shell command. You could use `make` as well.
cmd = "go build -o ../tmp/main ./ "
cmd = "go build -o ../tmp/main ./ "
# Binary file yields from `cmd`.
bin = "../tmp/main"
# Customize binary.

12
bin/docker-init.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/bin/bash
if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then
# start master
/bin/bash /app/bin/docker-start-master.sh
# start crawlab
crawlab-server master
else
# start crawlab
crawlab-server worker
fi

20
bin/docker-start-master.sh Executable file
View File

@@ -0,0 +1,20 @@
#!/bin/bash
# start nginx
service nginx start
# start seaweedfs server
seaweedfsDataPath=/data/seaweedfs
if [ -e ${seaweedfsDataPath} ]; then
:
else
mkdir -p ${seaweedfsDataPath}
fi
weed server \
-dir /data \
-master.dir ${seaweedfsDataPath} \
-volume.dir.idx ${seaweedfsDataPath} \
-ip localhost \
-volume.port 9999 \
-filer \
>> /var/log/weed.log 2>&1 &

73
docker_init.sh
View File

@@ -1,73 +0,0 @@
#!/bin/bash
# replace absolute api url to relative
jspath=`ls /app/dist/js/index.*.js`
sed -i "s?VUE_APP_API_BASE_URL?\/api?g" ${jspath}
# replace default api path to new one
# if [ "${CRAWLAB_API_ADDRESS}" = "" ];
# then
# :
# else
# jspath=`ls /app/dist/js/app.*.js`
# sed -i "s?###CRAWLAB_API_ADDRESS###?${CRAWLAB_API_ADDRESS}?g" ${jspath}
# fi
# replace base url
# if [ "${CRAWLAB_BASE_URL}" = "" ];
# then
# :
# else
# indexpath=/app/dist/index.html
# sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath}
# sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath}
# fi
#grant script
# chmod +x /app/backend/scripts/*.sh
# install languages
# if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_DOTNET}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ];
# then
# echo "installing languages"
# echo "you can view log at /var/log/install.sh.log"
# /bin/sh /app/backend/scripts/install.sh >> /var/log/install.sh.log 2>&1 &
# fi
# generate ssh
ssh-keygen -q -t rsa -N "" -f ${HOME}/.ssh/id_rsa
# ssh config
touch ${HOME}/.ssh/config && chmod 600 ${HOME}/.ssh/config
cat > ${HOME}/.ssh/config <<EOF
Host *
StrictHostKeyChecking no
EOF
# start backend
if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then
# start nginx
service nginx start
# start seaweedfs server
seaweedfsDataPath=/data/seaweedfs
if [ -e ${seaweedfsDataPath} ]; then
:
else
mkdir -p ${seaweedfsDataPath}
fi
weed server \
-dir /data \
-master.dir ${seaweedfsDataPath} \
-volume.dir.idx ${seaweedfsDataPath} \
-ip localhost \
-volume.port 9999 \
-filer \
>> /var/log/weed.log 2>&1 &
# start crawlab
crawlab-server master
else
# start crawlab
crawlab-server worker
fi

6
workspace/docker-compose.yml
View File

@@ -3,17 +3,21 @@ services:
master:
build:
context: ./dockerfiles/golang
command: "air -c .air.master.conf"
command: /bin/bash -c "/app/bin/docker-start-master.sh && air -c .air.master.conf"
volumes:
- "./.crawlab/master:/root/.crawlab"
- ../backend:/backend
- ../backend/go.mod.local:/backend/go.mod
- ../..:/libs/crawlab-team
- ../bin:/app/bin
- ../nginx:/etc/nginx/conf.d
- ../frontend/dist:/app/dist
environment:
CRAWLAB_NODE_MASTER: "Y"
CRAWLAB_MONGO_HOST: "mongo"
ports:
- "9080:8080"
- "9000:8000"
depends_on:
- mongo

3
workspace/dockerfiles/node/Dockerfile
View File

@@ -1,5 +1,4 @@
FROM node:latest
FROM node:12
WORKDIR frontend
ENV SASS_BINARY_SITE=https://npm.taobao.org/mirrors/node-sass/
RUN npm config set registry "http://registry.npm.taobao.org"