From 835713b73390c45e29b642b62a1c23d64f50a6f2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:11:49 +0800 Subject: [PATCH 01/13] updated Dockerfile --- Dockerfile | 39 +++++++++++++++++++ crawlab.conf | 5 +++ crawlab/Dockerfile | 26 ------------- .../docker-compose.yml => docker-compose.yml | 0 4 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 Dockerfile create mode 100644 crawlab.conf delete mode 100644 crawlab/Dockerfile rename crawlab/docker-compose.yml => docker-compose.yml (100%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d1a71f77 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# images +FROM ubuntu:latest + +# source files +ADD . /opt/crawlab + +# install python +RUN apt-get update +RUN apt-get install -y python3 python3-pip net-tools iputils-ping vim ntp git nginx + +# soft link +RUN ln -s /usr/bin/pip3 /usr/local/bin/pip +RUN ln -s /usr/bin/python3 /usr/local/bin/python + +# install backend +RUN pip install -U setuptools +RUN pip install -r /opt/crawlab/crawlab/requirements.txt + +# install nvm +RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash +RUN export NVM_DIR="$HOME/.nvm" +RUN [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm +RUN nvm install 8.12 +RUN nvm use 8.12 + +# install frontend +WORKDIR /opt/crawlab/frontend +RUN npm install -g yarn pm2 +RUN yarn install + +# nginx config & start frontend +RUN cp /opt/crawlab/crawlab.conf /etc/nginx/conf.d +RUN service nginx reload + +# start backend +WORKDIR /opt/crawlab/crawlab +CMD pm2 start app.py +CMD pm2 start flower.py +CMD pm2 start worker.py diff --git a/crawlab.conf b/crawlab.conf new file mode 100644 index 00000000..f0b7cef2 --- /dev/null +++ b/crawlab.conf @@ -0,0 +1,5 @@ +server { + listen 8080; + root /opt/crawlab/frontend/dist; + index index.html; +} \ No newline at end of file diff --git a/crawlab/Dockerfile b/crawlab/Dockerfile deleted file mode 100644 index b286d956..00000000 --- a/crawlab/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# images -#FROM python:latest -FROM ubuntu:latest - -# source files -ADD . /opt/crawlab - -# add dns -RUN cat /etc/resolv.conf - -# install python -RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping vim ntp - -# soft link -RUN ln -s /usr/bin/pip3 /usr/local/bin/pip -RUN ln -s /usr/bin/python3 /usr/local/bin/python - -# install required libraries -RUN pip install -U setuptools -RUN pip install -r /opt/crawlab/requirements.txt - -# execute apps -WORKDIR /opt/crawlab -CMD python ./bin/run_worker.py -CMD python app.py diff --git a/crawlab/docker-compose.yml b/docker-compose.yml similarity index 100% rename from crawlab/docker-compose.yml rename to docker-compose.yml From 25f257440a646fb3ca8038b29ecfb067388fd243 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:19:44 +0800 Subject: [PATCH 02/13] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d1a71f77..b512f630 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping vim ntp git nginx +RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx # soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip From 026f22f2f784418f745445a2b54ac6df5c45ae1e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:23:50 +0800 Subject: [PATCH 03/13] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b512f630..3c06a648 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN ln -s /usr/bin/python3 /usr/local/bin/python # install backend RUN pip install -U setuptools -RUN pip install -r /opt/crawlab/crawlab/requirements.txt +RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple # install nvm RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash From 128c7ebce809c9c1b39920475756bbc8383f5946 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:28:14 +0800 Subject: [PATCH 04/13] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3c06a648..8a9831b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx +RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx curl # soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip From 33e0eb4b5ce65f7318709b840ac9aadcf6b3da69 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:38:37 +0800 Subject: [PATCH 05/13] updated Dockerfile --- Dockerfile | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a9831b4..9fc53ede 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,27 +6,33 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx curl +RUN apt-get install -y python3 python3-pip net-tools iputils-ping redis-server git nginx curl -# soft link +# python soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python +# install mongodb +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 +echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list +apt-get update +apt-get install -y mongodb-org + # install backend RUN pip install -U setuptools RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple # install nvm -RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash -RUN export NVM_DIR="$HOME/.nvm" -RUN [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm -RUN nvm install 8.12 -RUN nvm use 8.12 +#RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash +#RUN export NVM_DIR="$HOME/.nvm" +#RUN [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm +#RUN nvm install 8.12 +#RUN nvm use 8.12 # install frontend WORKDIR /opt/crawlab/frontend -RUN npm install -g yarn pm2 -RUN yarn install +#RUN npm install -g yarn pm2 +#RUN yarn install # nginx config & start frontend RUN cp /opt/crawlab/crawlab.conf /etc/nginx/conf.d @@ -34,6 +40,9 @@ RUN service nginx reload # start backend WORKDIR /opt/crawlab/crawlab -CMD pm2 start app.py -CMD pm2 start flower.py -CMD pm2 start worker.py +CMD python app.py +CMD python flower.py +CMD python worker.py +#CMD pm2 start app.py +#CMD pm2 start flower.py +#CMD pm2 start worker.py From 452b85eb0a19f63a7807e36e75832f8338792dab Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:39:05 +0800 Subject: [PATCH 06/13] updated Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 9fc53ede..f5ed3d09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DC echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list apt-get update apt-get install -y mongodb-org +service mongod start # install backend RUN pip install -U setuptools From a7811575897754fc069789064d11d2075dcba27c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:40:16 +0800 Subject: [PATCH 07/13] updated Dockerfile --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f5ed3d09..3d0e0b71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,11 @@ RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python # install mongodb -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list -apt-get update -apt-get install -y mongodb-org -service mongod start +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 +RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list +RUN apt-get update +RUN apt-get install -y mongodb-org +RUN service mongod start # install backend RUN pip install -U setuptools From 35e942d024fdd9592f32a778f58bdb2c2dad40db Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:42:13 +0800 Subject: [PATCH 08/13] updated Dockerfile --- .dockerignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6d9fa983 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.idea +logs +*.log +node_modules \ No newline at end of file From ce5e34c23389eb2bc9b080f7082a7be23c326ca2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:46:21 +0800 Subject: [PATCH 09/13] updated Dockerfile --- .dockerignore | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6d9fa983..091ef16c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ .idea logs *.log -node_modules \ No newline at end of file +**/node_modules \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3d0e0b71..eb29a412 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN ln -s /usr/bin/python3 /usr/local/bin/python # install mongodb RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list +RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update RUN apt-get install -y mongodb-org RUN service mongod start From 393d0dde5b1d4388f2e3cfa7163236058076edde Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:53:48 +0800 Subject: [PATCH 10/13] updated Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index eb29a412..45d415d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ # images FROM ubuntu:latest +# set as non-interactive +RUN DEBIAN_FRONTEND=noninteractive + # source files ADD . /opt/crawlab @@ -13,6 +16,7 @@ RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python # install mongodb +RUN echo "Asia/Shanghai" > /etc/timezone && dpkg-reconfigure -f noninteractive tzdata RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update From 433065275536c0223cc67d05e227f11bcc069268 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:54:00 +0800 Subject: [PATCH 11/13] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 45d415d0..ef38e1b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping redis-server git nginx curl +RUN apt-get install -y python3 python3-pip net-tools iputils-ping redis-server git nginx ntp curl # python soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip From 4a6dca811cb7fa1a98879c188cb5ef80057fce36 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:57:32 +0800 Subject: [PATCH 12/13] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef38e1b5..2987261c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:latest # set as non-interactive -RUN DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND noninteractive # source files ADD . /opt/crawlab From 1ac129c32c26d2220c80d1314577fe34fe34c78b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 08:01:58 +0800 Subject: [PATCH 13/13] updated Dockerfile --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2987261c..f53aa906 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,6 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75 RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update RUN apt-get install -y mongodb-org -RUN service mongod start # install backend RUN pip install -U setuptools @@ -51,3 +50,9 @@ CMD python worker.py #CMD pm2 start app.py #CMD pm2 start flower.py #CMD pm2 start worker.py + +# start mongodb +CMD mongod + +# start redis +CMD redis-server