From 19cd332f86fc68cf335739a4ba4924b40b2c4ce7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:11:49 +0800 Subject: [PATCH 01/80] updated Dockerfile --- Dockerfile | 39 +++++++++++++++++++ crawlab.conf | 5 +++ crawlab/Dockerfile | 26 ------------- .../docker-compose.yml => docker-compose.yml | 0 4 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 Dockerfile create mode 100644 crawlab.conf delete mode 100644 crawlab/Dockerfile rename crawlab/docker-compose.yml => docker-compose.yml (100%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d1a71f77 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# images +FROM ubuntu:latest + +# source files +ADD . /opt/crawlab + +# install python +RUN apt-get update +RUN apt-get install -y python3 python3-pip net-tools iputils-ping vim ntp git nginx + +# soft link +RUN ln -s /usr/bin/pip3 /usr/local/bin/pip +RUN ln -s /usr/bin/python3 /usr/local/bin/python + +# install backend +RUN pip install -U setuptools +RUN pip install -r /opt/crawlab/crawlab/requirements.txt + +# install nvm +RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash +RUN export NVM_DIR="$HOME/.nvm" +RUN [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm +RUN nvm install 8.12 +RUN nvm use 8.12 + +# install frontend +WORKDIR /opt/crawlab/frontend +RUN npm install -g yarn pm2 +RUN yarn install + +# nginx config & start frontend +RUN cp /opt/crawlab/crawlab.conf /etc/nginx/conf.d +RUN service nginx reload + +# start backend +WORKDIR /opt/crawlab/crawlab +CMD pm2 start app.py +CMD pm2 start flower.py +CMD pm2 start worker.py diff --git a/crawlab.conf b/crawlab.conf new file mode 100644 index 00000000..f0b7cef2 --- /dev/null +++ b/crawlab.conf @@ -0,0 +1,5 @@ +server { + listen 8080; + root /opt/crawlab/frontend/dist; + index index.html; +} \ No newline at end of file diff --git a/crawlab/Dockerfile b/crawlab/Dockerfile deleted file mode 100644 index b286d956..00000000 --- a/crawlab/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# images -#FROM python:latest -FROM ubuntu:latest - -# source files -ADD . /opt/crawlab - -# add dns -RUN cat /etc/resolv.conf - -# install python -RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping vim ntp - -# soft link -RUN ln -s /usr/bin/pip3 /usr/local/bin/pip -RUN ln -s /usr/bin/python3 /usr/local/bin/python - -# install required libraries -RUN pip install -U setuptools -RUN pip install -r /opt/crawlab/requirements.txt - -# execute apps -WORKDIR /opt/crawlab -CMD python ./bin/run_worker.py -CMD python app.py diff --git a/crawlab/docker-compose.yml b/docker-compose.yml similarity index 100% rename from crawlab/docker-compose.yml rename to docker-compose.yml From cf7e175802b1c3bd397bebff399649d6deb61850 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:19:44 +0800 Subject: [PATCH 02/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d1a71f77..b512f630 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping vim ntp git nginx +RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx # soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip From 2fd48e7b5a9102167f18bd293d50b00458e47e75 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:23:50 +0800 Subject: [PATCH 03/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b512f630..3c06a648 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN ln -s /usr/bin/python3 /usr/local/bin/python # install backend RUN pip install -U setuptools -RUN pip install -r /opt/crawlab/crawlab/requirements.txt +RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple # install nvm RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash From a69e7129a37a462f1b5a3de514c9bcccae31af5b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:28:14 +0800 Subject: [PATCH 04/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3c06a648..8a9831b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx +RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx curl # soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip From 43b313dc63a189decf5778a4a691a3d3d801a187 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:38:37 +0800 Subject: [PATCH 05/80] updated Dockerfile --- Dockerfile | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a9831b4..9fc53ede 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,27 +6,33 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx curl +RUN apt-get install -y python3 python3-pip net-tools iputils-ping redis-server git nginx curl -# soft link +# python soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python +# install mongodb +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 +echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list +apt-get update +apt-get install -y mongodb-org + # install backend RUN pip install -U setuptools RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple # install nvm -RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash -RUN export NVM_DIR="$HOME/.nvm" -RUN [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm -RUN nvm install 8.12 -RUN nvm use 8.12 +#RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash +#RUN export NVM_DIR="$HOME/.nvm" +#RUN [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm +#RUN nvm install 8.12 +#RUN nvm use 8.12 # install frontend WORKDIR /opt/crawlab/frontend -RUN npm install -g yarn pm2 -RUN yarn install +#RUN npm install -g yarn pm2 +#RUN yarn install # nginx config & start frontend RUN cp /opt/crawlab/crawlab.conf /etc/nginx/conf.d @@ -34,6 +40,9 @@ RUN service nginx reload # start backend WORKDIR /opt/crawlab/crawlab -CMD pm2 start app.py -CMD pm2 start flower.py -CMD pm2 start worker.py +CMD python app.py +CMD python flower.py +CMD python worker.py +#CMD pm2 start app.py +#CMD pm2 start flower.py +#CMD pm2 start worker.py From de69a66bdba37356c672a82aef4df9c2cbdb1489 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:39:05 +0800 Subject: [PATCH 06/80] updated Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 9fc53ede..f5ed3d09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DC echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list apt-get update apt-get install -y mongodb-org +service mongod start # install backend RUN pip install -U setuptools From a4c19fd60e934efbd34a16a2b3f740997018a27c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:40:16 +0800 Subject: [PATCH 07/80] updated Dockerfile --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f5ed3d09..3d0e0b71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,11 @@ RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python # install mongodb -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list -apt-get update -apt-get install -y mongodb-org -service mongod start +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 +RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list +RUN apt-get update +RUN apt-get install -y mongodb-org +RUN service mongod start # install backend RUN pip install -U setuptools From 32e2c61cf85151680d670a7e6448e4654ab71604 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:42:13 +0800 Subject: [PATCH 08/80] updated Dockerfile --- .dockerignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6d9fa983 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.idea +logs +*.log +node_modules \ No newline at end of file From 1f22b05219f5c651a7e794297c90052cd50562bd Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:46:21 +0800 Subject: [PATCH 09/80] updated Dockerfile --- .dockerignore | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6d9fa983..091ef16c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ .idea logs *.log -node_modules \ No newline at end of file +**/node_modules \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3d0e0b71..eb29a412 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN ln -s /usr/bin/python3 /usr/local/bin/python # install mongodb RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list +RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update RUN apt-get install -y mongodb-org RUN service mongod start From 203e1b899318cba862d5107548ab7528f2408245 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:53:48 +0800 Subject: [PATCH 10/80] updated Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index eb29a412..45d415d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ # images FROM ubuntu:latest +# set as non-interactive +RUN DEBIAN_FRONTEND=noninteractive + # source files ADD . /opt/crawlab @@ -13,6 +16,7 @@ RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python # install mongodb +RUN echo "Asia/Shanghai" > /etc/timezone && dpkg-reconfigure -f noninteractive tzdata RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update From 5ba1102f975208ee742e4196a4ffa8a6a32645bb Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:54:00 +0800 Subject: [PATCH 11/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 45d415d0..ef38e1b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping redis-server git nginx curl +RUN apt-get install -y python3 python3-pip net-tools iputils-ping redis-server git nginx ntp curl # python soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip From 99f60a29245d51db68b9f9ec4985947e034ea780 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 07:57:32 +0800 Subject: [PATCH 12/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef38e1b5..2987261c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:latest # set as non-interactive -RUN DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND noninteractive # source files ADD . /opt/crawlab From 854da26e81b4ac0327245a68921a3faecbde81ad Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 08:01:58 +0800 Subject: [PATCH 13/80] updated Dockerfile --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2987261c..f53aa906 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,6 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75 RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update RUN apt-get install -y mongodb-org -RUN service mongod start # install backend RUN pip install -U setuptools @@ -51,3 +50,9 @@ CMD python worker.py #CMD pm2 start app.py #CMD pm2 start flower.py #CMD pm2 start worker.py + +# start mongodb +CMD mongod + +# start redis +CMD redis-server From 1697e719ea4d10591dd2b0e199bdf19d738b815f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 17:24:00 +0800 Subject: [PATCH 14/80] updated spiders --- crawlab/routes/spiders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index 91ab8608..a0a902da 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -509,7 +509,7 @@ class SpiderApi(BaseApi): }, r.status_code # get html parse tree - sel = etree.HTML(r.content) + sel = etree.HTML(r.content.decode('utf-8')) # remove unnecessary tags unnecessary_tags = [ From 61c421094f12e5b0bce6c76483de9cc2d12e1a72 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 17:53:07 +0800 Subject: [PATCH 15/80] updated spiders --- crawlab/routes/spiders.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index a0a902da..04e1eb96 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -509,7 +509,10 @@ class SpiderApi(BaseApi): }, r.status_code # get html parse tree - sel = etree.HTML(r.content.decode('utf-8')) + try: + sel = etree.HTML(r.content.decode('utf-8')) + except Exception as err: + sel = etree.HTML(r.content) # remove unnecessary tags unnecessary_tags = [ From bee8e3353641bac2b4970485c5ac3c5e42271f03 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 20:15:31 +0800 Subject: [PATCH 16/80] updated Dockerfile --- Dockerfile | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index f53aa906..03f07031 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,11 @@ FROM ubuntu:latest # set as non-interactive ENV DEBIAN_FRONTEND noninteractive +# environment variables +ENV NVM_DIR /usr/local/nvm +ENV NODE_VERSION 8.12 +ENV WORK_DIR /opt/crawlab + # source files ADD . /opt/crawlab @@ -27,32 +32,34 @@ RUN pip install -U setuptools RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple # install nvm -#RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash -#RUN export NVM_DIR="$HOME/.nvm" -#RUN [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm -#RUN nvm install 8.12 -#RUN nvm use 8.12 +RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ + && source $NVM_DIR/nvm.sh \ + && nvm install v$NODE_VERSION \ + && nvm use v$NODE_VERSION \ + && nvm alias default v$NODE_VERSION # install frontend -WORKDIR /opt/crawlab/frontend -#RUN npm install -g yarn pm2 -#RUN yarn install +RUN npm install -g yarn pm2 +RUN cd /opt/crawlab/frontend && yarn install # nginx config & start frontend -RUN cp /opt/crawlab/crawlab.conf /etc/nginx/conf.d +RUN cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d RUN service nginx reload -# start backend -WORKDIR /opt/crawlab/crawlab -CMD python app.py -CMD python flower.py -CMD python worker.py -#CMD pm2 start app.py -#CMD pm2 start flower.py -#CMD pm2 start worker.py - # start mongodb CMD mongod # start redis CMD redis-server + +# start backend +WORKDIR /opt/crawlab/crawlab +CMD python $WORK_DIR/crawlab/app.py +CMD python $WORK_DIR/crawlab/flower.py +CMD python $WORK_DIR/crawlab/worker.py +#CMD pm2 start $WORK_DIR/crawlab/app.py +#CMD pm2 start $WORK_DIR/crawlab/flower.py +#CMD pm2 start $WORK_DIR/crawlab/worker.py + +EXPOSE 8080 +EXPOSE 8000 \ No newline at end of file From f9a8d674eabc4afec4abced213260ea597bfaff2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 20:23:01 +0800 Subject: [PATCH 17/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 03f07031..a8eaa037 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update RUN apt-get install -y mongodb-org # install backend -RUN pip install -U setuptools +RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple # install nvm From 378677d579ae3678c729a9f658c6e8bbb98e6128 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 20:24:21 +0800 Subject: [PATCH 18/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a8eaa037..778dbe34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.ts # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ - && source $NVM_DIR/nvm.sh \ + && . $NVM_DIR/nvm.sh \ && nvm install v$NODE_VERSION \ && nvm use v$NODE_VERSION \ && nvm alias default v$NODE_VERSION From 6cc4164c134453dbe81539196d8d903be7d51a94 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 20:35:47 +0800 Subject: [PATCH 19/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 778dbe34..8a138c14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.ts # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ - && . $NVM_DIR/nvm.sh \ + && bash $NVM_DIR/nvm.sh \ && nvm install v$NODE_VERSION \ && nvm use v$NODE_VERSION \ && nvm alias default v$NODE_VERSION From cc51592ebcbe6cc6a6e32d29bbcfe6281fe79815 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 20:41:16 +0800 Subject: [PATCH 20/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8a138c14..778dbe34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.ts # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ - && bash $NVM_DIR/nvm.sh \ + && . $NVM_DIR/nvm.sh \ && nvm install v$NODE_VERSION \ && nvm use v$NODE_VERSION \ && nvm alias default v$NODE_VERSION From 1c41ee3063835432056f25bca74b270317c9f8b7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 20:43:27 +0800 Subject: [PATCH 21/80] updated Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 778dbe34..6997888b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,8 @@ RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | b && nvm install v$NODE_VERSION \ && nvm use v$NODE_VERSION \ && nvm alias default v$NODE_VERSION +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/v$NODE_VERSION/bin:$PATH # install frontend RUN npm install -g yarn pm2 From 276a393caec2cf047c658d4fcc27dd4cb995d304 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 6 Jun 2019 20:50:27 +0800 Subject: [PATCH 22/80] updated Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6997888b..5aebb197 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,8 +37,8 @@ RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | b && nvm install v$NODE_VERSION \ && nvm use v$NODE_VERSION \ && nvm alias default v$NODE_VERSION -ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules -ENV PATH $NVM_DIR/v$NODE_VERSION/bin:$PATH +RUN export NODE_PATH=$NVM_DIR/v$NODE_VERSION/lib/node_modules +RUN export PATH=$NVM_DIR/v$NODE_VERSION/bin:$PATH # install frontend RUN npm install -g yarn pm2 From d006a44db532f38fe6818688ecdcccd9e1764c00 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 16:11:39 +0800 Subject: [PATCH 23/80] hide deploys page --- frontend/src/router/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 0bfae74f..e03e9f49 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -159,6 +159,7 @@ export const constantRouterMap = [ name: 'Deploy', path: '/deploys', component: Layout, + hidden: true, meta: { title: 'Deploy', icon: 'fa fa-cloud' From 25565d0f8771cf9a9d7aea0ef03c5ff0c228014c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 16:17:49 +0800 Subject: [PATCH 24/80] edited config.py --- crawlab/config/config.py | 2 +- crawlab/tasks/spider.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crawlab/config/config.py b/crawlab/config/config.py index e235d400..58f1a85b 100644 --- a/crawlab/config/config.py +++ b/crawlab/config/config.py @@ -13,7 +13,7 @@ PYTHON_ENV_PATH = '/Users/yeqing/.pyenv/shims/python' PROJECT_DEPLOY_FILE_FOLDER = '/var/crawlab' # 爬虫日志路径 -PROJECT_LOGS_FOLDER = '../deployfile/logs' +PROJECT_LOGS_FOLDER = '/var/log/crawlab' # 打包临时文件夹 PROJECT_TMP_FOLDER = '/tmp' diff --git a/crawlab/tasks/spider.py b/crawlab/tasks/spider.py index 3bdc65bc..a568dc0f 100644 --- a/crawlab/tasks/spider.py +++ b/crawlab/tasks/spider.py @@ -6,7 +6,7 @@ from time import sleep from bson import ObjectId from pymongo import ASCENDING, DESCENDING -from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, PYTHON_ENV_PATH, MONGO_HOST, MONGO_PORT, MONGO_DB +from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, MONGO_HOST, MONGO_PORT, MONGO_DB from constants.task import TaskStatus from db.manager import db_manager from .celery import celery_app From 9cb1e1c0ec0095c94c09b5350bbf666e787c86c4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 16:18:00 +0800 Subject: [PATCH 25/80] edited config.py --- crawlab/config/config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crawlab/config/config.py b/crawlab/config/config.py index 58f1a85b..b01810ef 100644 --- a/crawlab/config/config.py +++ b/crawlab/config/config.py @@ -5,11 +5,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__fil # 爬虫源码路径 PROJECT_SOURCE_FILE_FOLDER = os.path.join(BASE_DIR, "spiders") -# 配置python虚拟环境的路径 -PYTHON_ENV_PATH = '/Users/yeqing/.pyenv/shims/python' - # 爬虫部署路径 -# PROJECT_DEPLOY_FILE_FOLDER = '../deployfile' PROJECT_DEPLOY_FILE_FOLDER = '/var/crawlab' # 爬虫日志路径 From f9740cb049ce621650e24eb46dc1f5babf7cc3b5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 16:20:11 +0800 Subject: [PATCH 26/80] edited config.py --- crawlab/config/config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crawlab/config/config.py b/crawlab/config/config.py index b01810ef..da185c57 100644 --- a/crawlab/config/config.py +++ b/crawlab/config/config.py @@ -32,11 +32,6 @@ CELERY_TIMEZONE = 'Asia/Shanghai' # 是否启用UTC CELERY_ENABLE_UTC = True -# Celery Scheduler Redis URL -CELERY_BEAT_SCHEDULER = 'utils.redisbeat.RedisScheduler' -CELERY_REDIS_SCHEDULER_URL = 'redis://localhost:6379' -CELERY_REDIS_SCHEDULER_KEY = 'celery:beat:order_tasks' - # flower variables FLOWER_API_ENDPOINT = 'http://localhost:5555/api' From 75463728e552842829ebb470bbf63bf474024f1a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 16:44:40 +0800 Subject: [PATCH 27/80] modified fields extraction algorithm --- crawlab/routes/spiders.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index 04e1eb96..14d028e6 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -657,19 +657,24 @@ class SpiderApi(BaseApi): # get list item selector item_selector = None + item_selector_type = 'css' if max_tag.get('id') is not None: item_selector = f'#{max_tag.get("id")} > {self._get_children(max_tag)[0].tag}' elif max_tag.get('class') is not None: cls_str = '.'.join([x for x in max_tag.get("class").split(' ') if x != '']) if len(sel.cssselect(f'.{cls_str}')) == 1: item_selector = f'.{cls_str} > {self._get_children(max_tag)[0].tag}' + else: + item_selector = max_tag.getroottree().getpath(max_tag) + item_selector_type = 'xpath' # get list fields fields = [] if item_selector is not None: first_tag = self._get_children(max_tag)[0] for i, tag in enumerate(self._get_text_child_tags(first_tag)): - if len(first_tag.cssselect(f'{tag.tag}')) == 1: + el_list = first_tag.cssselect(f'{tag.tag}') + if len(el_list) == 1: fields.append({ 'name': f'field{i + 1}', 'type': 'css', @@ -685,6 +690,15 @@ class SpiderApi(BaseApi): 'extract_type': 'text', 'query': f'{tag.tag}.{cls_str}', }) + else: + for j, el in enumerate(el_list): + if tag == el: + fields.append({ + 'name': f'field{i + 1}', + 'type': 'css', + 'extract_type': 'text', + 'query': f'{tag.tag}:nth-of-type({j + 1})', + }) for i, tag in enumerate(self._get_a_child_tags(self._get_children(max_tag)[0])): # if the tag is , extract its href @@ -710,6 +724,7 @@ class SpiderApi(BaseApi): return { 'status': 'ok', 'item_selector': item_selector, + 'item_selector_type': item_selector_type, 'pagination_selector': pagination_selector, 'fields': fields } From c48a92690784be91208d2ed80d9e1a13cdea6a50 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 16:51:03 +0800 Subject: [PATCH 28/80] pagination change --- crawlab/routes/spiders.py | 1 + frontend/src/components/Config/ConfigList.vue | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index 14d028e6..2cc39fea 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -553,6 +553,7 @@ class SpiderApi(BaseApi): '下页', 'next page', 'next', + '>' ] for tag in sel.iter(): if tag.text is not None and tag.text.lower().strip() in next_page_text_list: diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 0362b72f..56dc01c0 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -255,7 +255,9 @@ export default { .then(response => { if (response.data.item_selector) { this.$set(this.spiderForm, 'item_selector', response.data.item_selector) - this.$set(this.spiderForm, 'item_selector_type', 'css') + } + if (response.data.item_selector_type) { + this.$set(this.spiderForm, 'item_selector_type', response.data.item_selector_type) } if (response.data.fields && response.data.fields.length) { From 81f729646a0581761702db2d4ae9a3f86dcaeda4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 17:01:17 +0800 Subject: [PATCH 29/80] pagination change --- crawlab/spiders/spiders/spiders/config_spider.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crawlab/spiders/spiders/spiders/config_spider.py b/crawlab/spiders/spiders/spiders/config_spider.py index 13fa82bf..70693c0d 100644 --- a/crawlab/spiders/spiders/spiders/config_spider.py +++ b/crawlab/spiders/spiders/spiders/config_spider.py @@ -73,7 +73,10 @@ def get_next_url(response): if next_url is not None: if not next_url.startswith('http') and not next_url.startswith('//'): u = urlparse(response.url) - next_url = f'{u.scheme}://{u.netloc}{next_url}' + if next_url.startswith('/'): + next_url = f'{u.scheme}://{u.netloc}{next_url}' + else: + next_url = f'{u.scheme}://{u.netloc}{u.path}/{next_url}' return next_url return None From 158a048bae126d0e401f3b1c55983dee6b154896 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 17:11:42 +0800 Subject: [PATCH 30/80] pagination change --- crawlab/spiders/spiders/spiders/config_spider.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crawlab/spiders/spiders/spiders/config_spider.py b/crawlab/spiders/spiders/spiders/config_spider.py index 70693c0d..77e65862 100644 --- a/crawlab/spiders/spiders/spiders/config_spider.py +++ b/crawlab/spiders/spiders/spiders/config_spider.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import os import sys -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin import scrapy @@ -72,11 +72,8 @@ def get_next_url(response): # found next url if next_url is not None: if not next_url.startswith('http') and not next_url.startswith('//'): - u = urlparse(response.url) - if next_url.startswith('/'): - next_url = f'{u.scheme}://{u.netloc}{next_url}' - else: - next_url = f'{u.scheme}://{u.netloc}{u.path}/{next_url}' + return urljoin(response.url, next_url) + else: return next_url return None From 2abd8df9d7c535c09b7eb2b222fb3becbeb6fb3b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 19:06:37 +0800 Subject: [PATCH 31/80] added virtual scroll list for log view --- frontend/package.json | 1 + frontend/src/components/Charts/keyboard.vue | 156 --------- frontend/src/components/Charts/lineMarker.vue | 227 ------------- frontend/src/components/Charts/mixChart.vue | 271 ---------------- .../src/components/Charts/mixins/resize.js | 32 -- frontend/src/components/DragSelect/index.vue | 61 ---- frontend/src/components/Dropzone/index.vue | 297 ------------------ .../src/components/ScrollView/LogItem.vue | 39 +++ .../src/components/ScrollView/LogView.vue | 78 +++++ frontend/src/views/task/TaskDetail.vue | 8 +- frontend/yarn.lock | 5 + 11 files changed, 126 insertions(+), 1049 deletions(-) delete mode 100644 frontend/src/components/Charts/keyboard.vue delete mode 100644 frontend/src/components/Charts/lineMarker.vue delete mode 100644 frontend/src/components/Charts/mixChart.vue delete mode 100644 frontend/src/components/Charts/mixins/resize.js delete mode 100644 frontend/src/components/DragSelect/index.vue delete mode 100644 frontend/src/components/Dropzone/index.vue create mode 100644 frontend/src/components/ScrollView/LogItem.vue create mode 100644 frontend/src/components/ScrollView/LogView.vue diff --git a/frontend/package.json b/frontend/package.json index e751c9f9..42bcf078 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -27,6 +27,7 @@ "vue-codemirror-lite": "^1.0.4", "vue-i18n": "^8.9.0", "vue-router": "^3.0.1", + "vue-virtual-scroll-list": "^1.3.9", "vuex": "^3.0.1" }, "devDependencies": { diff --git a/frontend/src/components/Charts/keyboard.vue b/frontend/src/components/Charts/keyboard.vue deleted file mode 100644 index 857b26ae..00000000 --- a/frontend/src/components/Charts/keyboard.vue +++ /dev/null @@ -1,156 +0,0 @@ - - - diff --git a/frontend/src/components/Charts/lineMarker.vue b/frontend/src/components/Charts/lineMarker.vue deleted file mode 100644 index 5d41b022..00000000 --- a/frontend/src/components/Charts/lineMarker.vue +++ /dev/null @@ -1,227 +0,0 @@ - - - diff --git a/frontend/src/components/Charts/mixChart.vue b/frontend/src/components/Charts/mixChart.vue deleted file mode 100644 index a8e271ca..00000000 --- a/frontend/src/components/Charts/mixChart.vue +++ /dev/null @@ -1,271 +0,0 @@ - - - diff --git a/frontend/src/components/Charts/mixins/resize.js b/frontend/src/components/Charts/mixins/resize.js deleted file mode 100644 index c4c432f4..00000000 --- a/frontend/src/components/Charts/mixins/resize.js +++ /dev/null @@ -1,32 +0,0 @@ -import { debounce } from '@/utils' - -export default { - data() { - return { - sidebarElm: null - } - }, - mounted() { - this.__resizeHandler = debounce(() => { - if (this.chart) { - this.chart.resize() - } - }, 100) - window.addEventListener('resize', this.__resizeHandler) - - this.sidebarElm = document.getElementsByClassName('sidebar-container')[0] - this.sidebarElm && this.sidebarElm.addEventListener('transitionend', this.sidebarResizeHandler) - }, - beforeDestroy() { - window.removeEventListener('resize', this.__resizeHandler) - - this.sidebarElm && this.sidebarElm.removeEventListener('transitionend', this.sidebarResizeHandler) - }, - methods: { - sidebarResizeHandler(e) { - if (e.propertyName === 'width') { - this.__resizeHandler() - } - } - } -} diff --git a/frontend/src/components/DragSelect/index.vue b/frontend/src/components/DragSelect/index.vue deleted file mode 100644 index 513be006..00000000 --- a/frontend/src/components/DragSelect/index.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - - - diff --git a/frontend/src/components/Dropzone/index.vue b/frontend/src/components/Dropzone/index.vue deleted file mode 100644 index 15d811d9..00000000 --- a/frontend/src/components/Dropzone/index.vue +++ /dev/null @@ -1,297 +0,0 @@ - - - - - diff --git a/frontend/src/components/ScrollView/LogItem.vue b/frontend/src/components/ScrollView/LogItem.vue new file mode 100644 index 00000000..4a5aa168 --- /dev/null +++ b/frontend/src/components/ScrollView/LogItem.vue @@ -0,0 +1,39 @@ + + + + + diff --git a/frontend/src/components/ScrollView/LogView.vue b/frontend/src/components/ScrollView/LogView.vue new file mode 100644 index 00000000..75eff8be --- /dev/null +++ b/frontend/src/components/ScrollView/LogView.vue @@ -0,0 +1,78 @@ + + + + + diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index 92198773..053ef963 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -7,11 +7,7 @@ -
- 
-              {{taskLog}}
-
-
+ @@ -37,10 +33,12 @@ import { } from 'vuex' import TaskOverview from '../../components/Overview/TaskOverview' import GeneralTableView from '../../components/TableView/GeneralTableView' +import LogView from '../../components/ScrollView/LogView' export default { name: 'TaskDetail', components: { + LogView, GeneralTableView, TaskOverview }, diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 75d4b025..2d266229 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -8493,6 +8493,11 @@ vue-template-es2015-compiler@^1.6.0, vue-template-es2015-compiler@^1.8.2: version "1.8.2" resolved "http://registry.npm.taobao.org/vue-template-es2015-compiler/download/vue-template-es2015-compiler-1.8.2.tgz#dd73e80ba58bb65dd7a8aa2aeef6089cf6116f2a" +vue-virtual-scroll-list@^1.3.9: + version "1.3.9" + resolved "https://registry.npm.taobao.org/vue-virtual-scroll-list/download/vue-virtual-scroll-list-1.3.9.tgz#ba3ce6425374fb323ea83ab33daa2727117808ed" + integrity sha1-ujzmQlN0+zI+qDqzPaonJxF4CO0= + vue@^2.3.3: version "2.6.10" resolved "https://registry.npm.taobao.org/vue/download/vue-2.6.10.tgz#a72b1a42a4d82a721ea438d1b6bf55e66195c637" From 71fc7cfc090f31aa993759e2ac0d47551830fdf0 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 19:31:14 +0800 Subject: [PATCH 32/80] updated Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5aebb197..688bb2ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,8 +41,8 @@ RUN export NODE_PATH=$NVM_DIR/v$NODE_VERSION/lib/node_modules RUN export PATH=$NVM_DIR/v$NODE_VERSION/bin:$PATH # install frontend -RUN npm install -g yarn pm2 -RUN cd /opt/crawlab/frontend && yarn install +RUN $NVM_DIR/v$NODE_VERSION/bin/npm install -g yarn pm2 +RUN cd /opt/crawlab/frontend && $NVM_DIR/v$NODE_VERSION/bin/yarn install # nginx config & start frontend RUN cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d From 797a6124cd229d775acfa12f0427607548559761 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 19:38:35 +0800 Subject: [PATCH 33/80] updated Dockerfile --- Dockerfile | 15 +-------------- docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 688bb2ea..0f5c2917 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,19 +14,12 @@ ADD . /opt/crawlab # install python RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping redis-server git nginx ntp curl +RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx ntp curl # python soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python -# install mongodb -RUN echo "Asia/Shanghai" > /etc/timezone && dpkg-reconfigure -f noninteractive tzdata -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list -RUN apt-get update -RUN apt-get install -y mongodb-org - # install backend RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple @@ -48,12 +41,6 @@ RUN cd /opt/crawlab/frontend && $NVM_DIR/v$NODE_VERSION/bin/yarn install RUN cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d RUN service nginx reload -# start mongodb -CMD mongod - -# start redis -CMD redis-server - # start backend WORKDIR /opt/crawlab/crawlab CMD python $WORK_DIR/crawlab/app.py diff --git a/docker-compose.yml b/docker-compose.yml index fe2bfb78..088fdbb8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,11 @@ version: '3.3' # 表示该 Docker-Compose 文件使用的是 Version 2 file services: - web: # 指定服务名称 + app: # 指定服务名称 build: . # 指定 Dockerfile 所在路径 ports: # 指定端口映射 - "5001:5000" task: - image: crawlab:v3 + image: crawlab:latest db: image: mongo restart: always From 40910e524649ea69167defa922103413218f1ae9 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 20:21:08 +0800 Subject: [PATCH 34/80] updated Dockerfile --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0f5c2917..421f1430 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive # environment variables ENV NVM_DIR /usr/local/nvm -ENV NODE_VERSION 8.12 +ENV NODE_VERSION 8.12.0 ENV WORK_DIR /opt/crawlab # source files @@ -30,8 +30,10 @@ RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | b && nvm install v$NODE_VERSION \ && nvm use v$NODE_VERSION \ && nvm alias default v$NODE_VERSION -RUN export NODE_PATH=$NVM_DIR/v$NODE_VERSION/lib/node_modules -RUN export PATH=$NVM_DIR/v$NODE_VERSION/bin:$PATH +ENV NODE_PATH $NVM_DIR/versions/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/v$NODE_VERSION/bin:$PATH + +RUN ls $NVM_DIR/versions/v$NODE_VERSION/bin # install frontend RUN $NVM_DIR/v$NODE_VERSION/bin/npm install -g yarn pm2 From a23f4b27336490c314e608baf7812bfc5068fa03 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 20:21:37 +0800 Subject: [PATCH 35/80] updated Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 421f1430..6c2d60e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,8 +36,8 @@ ENV PATH $NVM_DIR/versions/v$NODE_VERSION/bin:$PATH RUN ls $NVM_DIR/versions/v$NODE_VERSION/bin # install frontend -RUN $NVM_DIR/v$NODE_VERSION/bin/npm install -g yarn pm2 -RUN cd /opt/crawlab/frontend && $NVM_DIR/v$NODE_VERSION/bin/yarn install +RUN $NVM_DIR/versions/v$NODE_VERSION/bin/npm install -g yarn pm2 +RUN cd /opt/crawlab/frontend && $NVM_DIR/versions/v$NODE_VERSION/bin/yarn install # nginx config & start frontend RUN cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d From 4e974f53da989830ba6b0cf28bdc96837812b8af Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 20:23:08 +0800 Subject: [PATCH 36/80] updated Dockerfile --- Dockerfile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c2d60e1..4416dfde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ # images FROM ubuntu:latest +# source files +ADD . /opt/crawlab + # set as non-interactive ENV DEBIAN_FRONTEND noninteractive @@ -9,21 +12,6 @@ ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION 8.12.0 ENV WORK_DIR /opt/crawlab -# source files -ADD . /opt/crawlab - -# install python -RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx ntp curl - -# python soft link -RUN ln -s /usr/bin/pip3 /usr/local/bin/pip -RUN ln -s /usr/bin/python3 /usr/local/bin/python - -# install backend -RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple -RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ && . $NVM_DIR/nvm.sh \ @@ -39,6 +27,18 @@ RUN ls $NVM_DIR/versions/v$NODE_VERSION/bin RUN $NVM_DIR/versions/v$NODE_VERSION/bin/npm install -g yarn pm2 RUN cd /opt/crawlab/frontend && $NVM_DIR/versions/v$NODE_VERSION/bin/yarn install +# install python +RUN apt-get update +RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx ntp curl + +# python soft link +RUN ln -s /usr/bin/pip3 /usr/local/bin/pip +RUN ln -s /usr/bin/python3 /usr/local/bin/python + +# install backend +RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple + # nginx config & start frontend RUN cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d RUN service nginx reload From 5065e471db2646d9cfeb85e9d3601b6b5f1b1960 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 20:25:55 +0800 Subject: [PATCH 37/80] updated Dockerfile --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4416dfde..ae19932c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,11 @@ ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION 8.12.0 ENV WORK_DIR /opt/crawlab +# install curl git +RUN apt-get update +RUN apt-get install curl git + + # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ && . $NVM_DIR/nvm.sh \ @@ -28,8 +33,7 @@ RUN $NVM_DIR/versions/v$NODE_VERSION/bin/npm install -g yarn pm2 RUN cd /opt/crawlab/frontend && $NVM_DIR/versions/v$NODE_VERSION/bin/yarn install # install python -RUN apt-get update -RUN apt-get install -y python3 python3-pip net-tools iputils-ping git nginx ntp curl +RUN apt-get install -y python3 python3-pip net-tools iputils-ping nginx ntp # python soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip From 4615362d86629820de6f7843402618970ef65537 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 20:27:16 +0800 Subject: [PATCH 38/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ae19932c..8f481a6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ENV WORK_DIR /opt/crawlab # install curl git RUN apt-get update -RUN apt-get install curl git +RUN apt-get install -y curl git # install nvm From e8f43ff9b9da59b149b8ee5750aa32b366593af7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 21:13:11 +0800 Subject: [PATCH 39/80] updated Dockerfile --- Dockerfile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8f481a6e..a095cd4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,21 +16,18 @@ ENV WORK_DIR /opt/crawlab RUN apt-get update RUN apt-get install -y curl git - # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ && . $NVM_DIR/nvm.sh \ && nvm install v$NODE_VERSION \ && nvm use v$NODE_VERSION \ && nvm alias default v$NODE_VERSION -ENV NODE_PATH $NVM_DIR/versions/v$NODE_VERSION/lib/node_modules -ENV PATH $NVM_DIR/versions/v$NODE_VERSION/bin:$PATH - -RUN ls $NVM_DIR/versions/v$NODE_VERSION/bin +ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH # install frontend -RUN $NVM_DIR/versions/v$NODE_VERSION/bin/npm install -g yarn pm2 -RUN cd /opt/crawlab/frontend && $NVM_DIR/versions/v$NODE_VERSION/bin/yarn install +RUN npm install -g yarn pm2 +RUN cd /opt/crawlab/frontend && yarn install # install python RUN apt-get install -y python3 python3-pip net-tools iputils-ping nginx ntp From 03240d984e91df16df72c55114a430937cdb59c1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 21:21:55 +0800 Subject: [PATCH 40/80] updated Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a095cd4e..432f6e37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,8 @@ ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH # install frontend -RUN npm install -g yarn pm2 -RUN cd /opt/crawlab/frontend && yarn install +RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org +RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.taobao.org # install python RUN apt-get install -y python3 python3-pip net-tools iputils-ping nginx ntp From 3e6f82db0a142114c862fef307495c5ff8d9f430 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 21:50:22 +0800 Subject: [PATCH 41/80] updated Dockerfile --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 432f6e37..8a857545 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,12 +46,7 @@ RUN service nginx reload # start backend WORKDIR /opt/crawlab/crawlab -CMD python $WORK_DIR/crawlab/app.py -CMD python $WORK_DIR/crawlab/flower.py -CMD python $WORK_DIR/crawlab/worker.py -#CMD pm2 start $WORK_DIR/crawlab/app.py -#CMD pm2 start $WORK_DIR/crawlab/flower.py -#CMD pm2 start $WORK_DIR/crawlab/worker.py +ENTRYPOINT python $WORK_DIR/manage.py EXPOSE 8080 EXPOSE 8000 \ No newline at end of file From c1c86609de236a749db0642dfefbd524d03b0c07 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 22:00:44 +0800 Subject: [PATCH 42/80] updated Dockerfile --- Dockerfile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8a857545..ebe30c01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,17 @@ RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.tao # install python RUN apt-get install -y python3 python3-pip net-tools iputils-ping nginx ntp +# install redis +RUN apt-get install redis-server +RUN service redis-server start + +# install mongodb +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 +RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list +RUN apt-get update +RUN apt-get install -y mongodb-org +RUN service mongod start + # python soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python @@ -46,7 +57,9 @@ RUN service nginx reload # start backend WORKDIR /opt/crawlab/crawlab -ENTRYPOINT python $WORK_DIR/manage.py +ENTRYPOINT cd /opt/crawlab/crawlab/frontend \ + && npm run build:prod \ + && python $WORK_DIR/manage.py EXPOSE 8080 EXPOSE 8000 \ No newline at end of file From 6ea5c472014af5250f252c0567f0c8724fac398e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 22:16:08 +0800 Subject: [PATCH 43/80] updated Dockerfile --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ebe30c01..958d0463 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,9 @@ ENV WORK_DIR /opt/crawlab RUN apt-get update RUN apt-get install -y curl git +# install python +RUN apt-get install -y python3 python3-pip net-tools iputils-ping ntp + # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ && . $NVM_DIR/nvm.sh \ @@ -29,8 +32,8 @@ ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.taobao.org -# install python -RUN apt-get install -y python3 python3-pip net-tools iputils-ping nginx ntp +# install nginx +RUN apt-get install -y nginx # install redis RUN apt-get install redis-server From b55d70c4e54782a232408a11bd0227b93952b43a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 22:26:36 +0800 Subject: [PATCH 44/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 958d0463..41b012a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update RUN apt-get install -y curl git # install python -RUN apt-get install -y python3 python3-pip net-tools iputils-ping ntp +RUN apt-get install -y python python3 python3-pip net-tools iputils-ping ntp # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ From ac2e9993a38dd4a3a2fa37f348896c420238e059 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 22:32:44 +0800 Subject: [PATCH 45/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 41b012a8..5247162f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,7 @@ RUN service nginx reload WORKDIR /opt/crawlab/crawlab ENTRYPOINT cd /opt/crawlab/crawlab/frontend \ && npm run build:prod \ - && python $WORK_DIR/manage.py + && python3 $WORK_DIR/manage.py EXPOSE 8080 EXPOSE 8000 \ No newline at end of file From 1f0f6beb2d92aea5f76bd45828300b5b668a18e6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:06:50 +0800 Subject: [PATCH 46/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5247162f..c29766d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.tao RUN apt-get install -y nginx # install redis -RUN apt-get install redis-server +RUN apt-get install -y redis-server RUN service redis-server start # install mongodb From 626700365e1c4aed3bbd127c43c0d7d8c5340837 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:07:28 +0800 Subject: [PATCH 47/80] updated Dockerfile --- .dockerignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 091ef16c..30de240b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ .idea logs -*.log -**/node_modules \ No newline at end of file +*.log \ No newline at end of file From 7bef223d5916d24dbebcd9aa6af32b2177c9bd15 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:14:47 +0800 Subject: [PATCH 48/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c29766d2..f45ced34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN service redis-server start # install mongodb RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list +RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update RUN apt-get install -y mongodb-org RUN service mongod start From 4d9d470034e8cbd5932dbcf9e90400def6c3bf14 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:28:34 +0800 Subject: [PATCH 49/80] updated Dockerfile --- Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index f45ced34..50857c27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,13 @@ ENV WORK_DIR /opt/crawlab RUN apt-get update RUN apt-get install -y curl git +# install mongodb +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 +RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list +RUN apt-get update +RUN apt-get install -y mongodb-org +RUN service mongod start + # install python RUN apt-get install -y python python3 python3-pip net-tools iputils-ping ntp @@ -39,13 +46,6 @@ RUN apt-get install -y nginx RUN apt-get install -y redis-server RUN service redis-server start -# install mongodb -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list -RUN apt-get update -RUN apt-get install -y mongodb-org -RUN service mongod start - # python soft link RUN ln -s /usr/bin/pip3 /usr/local/bin/pip RUN ln -s /usr/bin/python3 /usr/local/bin/python From ef69761eecb670f65b4c4f8454d213ce433bc935 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:31:36 +0800 Subject: [PATCH 50/80] updated Dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 50857c27..19a786e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,9 +12,9 @@ ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION 8.12.0 ENV WORK_DIR /opt/crawlab -# install curl git +# install pkg RUN apt-get update -RUN apt-get install -y curl git +RUN apt-get install -y curl git net-tools iputils-ping ntp # install mongodb RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 @@ -24,7 +24,7 @@ RUN apt-get install -y mongodb-org RUN service mongod start # install python -RUN apt-get install -y python python3 python3-pip net-tools iputils-ping ntp +RUN apt-get install -y python python3 python3-pip # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ From 456333a2f0198e7f4826c5f1957e6472c75bb48a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:36:18 +0800 Subject: [PATCH 51/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 19a786e4..b632f1de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ENV WORK_DIR /opt/crawlab # install pkg RUN apt-get update -RUN apt-get install -y curl git net-tools iputils-ping ntp +RUN apt-get install -y curl git net-tools iputils-ping ntp gnupg2 # install mongodb RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 From c4535ec7013708c0f777e056045d0869f1040791 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:41:19 +0800 Subject: [PATCH 52/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b632f1de..f89e39a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get install -y curl git net-tools iputils-ping ntp gnupg2 # install mongodb RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list +echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update RUN apt-get install -y mongodb-org RUN service mongod start From aa5407d63a649d609b2d8f359f1929c3d42dd5aa Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:43:05 +0800 Subject: [PATCH 53/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f89e39a4..b450f1af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get install -y curl git net-tools iputils-ping ntp gnupg2 # install mongodb RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list +RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update RUN apt-get install -y mongodb-org RUN service mongod start From fe405cb2757ba1d769307f5dec50e3bc09fc753a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 9 Jun 2019 23:47:20 +0800 Subject: [PATCH 54/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b450f1af..a6810fac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75 RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list RUN apt-get update RUN apt-get install -y mongodb-org -RUN service mongod start +RUN mongod >> /var/log/mongod.log 2>&1 & # install python RUN apt-get install -y python python3 python3-pip From e48ae31fee5b91a859d61f665691aebfa5a29e7a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 07:35:19 +0800 Subject: [PATCH 55/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a6810fac..13e80452 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,7 @@ RUN service nginx reload # start backend WORKDIR /opt/crawlab/crawlab -ENTRYPOINT cd /opt/crawlab/crawlab/frontend \ +ENTRYPOINT cd /opt/crawlab/frontend \ && npm run build:prod \ && python3 $WORK_DIR/manage.py From 77c23214af4688a75d9b0582013912f779bef55d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 07:58:22 +0800 Subject: [PATCH 56/80] updated Dockerfile --- Dockerfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 13e80452..c4bab9b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,14 @@ RUN mongod >> /var/log/mongod.log 2>&1 & # install python RUN apt-get install -y python python3 python3-pip +RUN ln -s /usr/bin/pip3 /usr/local/bin/pip +RUN ln -s /usr/bin/python3 /usr/local/bin/python +#RUN git clone git://github.com/yyuu/pyenv.git ~/.pyenv +#ENV PYENV_ROOT "$HOME/.pyenv" +#ENV PATH "$PYENV_ROOT/bin:$PATH" +#RUN eval "$(pyenv init -)" +#RUN pyenv install 3.6.8 +#RUN pyenv local 3.6.8 # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ @@ -46,10 +54,6 @@ RUN apt-get install -y nginx RUN apt-get install -y redis-server RUN service redis-server start -# python soft link -RUN ln -s /usr/bin/pip3 /usr/local/bin/pip -RUN ln -s /usr/bin/python3 /usr/local/bin/python - # install backend RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple @@ -59,10 +63,7 @@ RUN cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d RUN service nginx reload # start backend -WORKDIR /opt/crawlab/crawlab -ENTRYPOINT cd /opt/crawlab/frontend \ - && npm run build:prod \ - && python3 $WORK_DIR/manage.py - EXPOSE 8080 -EXPOSE 8000 \ No newline at end of file +EXPOSE 8000 +WORKDIR /opt/crawlab +ENTRYPOINT ["/bin/sh", "docker_init.sh"] \ No newline at end of file From 46bc3a85e1a49c413a6edda29a04f9c19e9dd962 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 21:13:07 +0800 Subject: [PATCH 57/80] updated Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c4bab9b7..965818a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,4 +66,4 @@ RUN service nginx reload EXPOSE 8080 EXPOSE 8000 WORKDIR /opt/crawlab -ENTRYPOINT ["/bin/sh", "docker_init.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/sh", "/opt/crawlab/docker_init.sh"] \ No newline at end of file From d44561a6bd2e3a8966980ace8646c2868bb2bf1f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 21:27:27 +0800 Subject: [PATCH 58/80] updated Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 965818a0..2c18cc39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # images -FROM ubuntu:latest +FROM python:3.6 # source files ADD . /opt/crawlab @@ -24,9 +24,9 @@ RUN apt-get install -y mongodb-org RUN mongod >> /var/log/mongod.log 2>&1 & # install python -RUN apt-get install -y python python3 python3-pip -RUN ln -s /usr/bin/pip3 /usr/local/bin/pip -RUN ln -s /usr/bin/python3 /usr/local/bin/python +#RUN apt-get install -y python python3 python3-pip +#RUN ln -s /usr/bin/pip3 /usr/local/bin/pip +#RUN ln -s /usr/bin/python3 /usr/local/bin/python #RUN git clone git://github.com/yyuu/pyenv.git ~/.pyenv #ENV PYENV_ROOT "$HOME/.pyenv" #ENV PATH "$PYENV_ROOT/bin:$PATH" From 08e5c44b32325ee7d45e91cf800edfd32911c9ab Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 21:36:26 +0800 Subject: [PATCH 59/80] added frontend dockerfile --- frontend/Dockerfile | 29 +++++++++++++++++++++++++++++ frontend/conf/crawlab.conf | 5 +++++ frontend/conf/nginx.conf | 4 ++-- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 frontend/Dockerfile create mode 100644 frontend/conf/crawlab.conf diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..554a3b8a --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,29 @@ +# images +FROM nginx:latest + +# source files +ADD . /opt/crawlab/frontend + +# environment variables +ENV NVM_DIR /usr/local/nvm +ENV NODE_VERSION 8.12.0 +ENV WORK_DIR /opt/crawlab/frontend + +# install nvm +RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ + && . $NVM_DIR/nvm.sh \ + && nvm install v$NODE_VERSION \ + && nvm use v$NODE_VERSION \ + && nvm alias default v$NODE_VERSION +ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH + +# install frontend +RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org +RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.taobao.org + +# nginx config & start frontend +RUN cp $WORK_DIR/conf/crawlab.conf /etc/nginx/conf.d +RUN service nginx reload + +CMD ["npm", "run", "build:prod"] diff --git a/frontend/conf/crawlab.conf b/frontend/conf/crawlab.conf new file mode 100644 index 00000000..da8513f3 --- /dev/null +++ b/frontend/conf/crawlab.conf @@ -0,0 +1,5 @@ +server { + listen 8080; + root /opt/crawlab/frontend/dist; + index index.html; +} diff --git a/frontend/conf/nginx.conf b/frontend/conf/nginx.conf index 72c248d2..56e4cd80 100644 --- a/frontend/conf/nginx.conf +++ b/frontend/conf/nginx.conf @@ -8,8 +8,8 @@ http { default_type application/octet-stream; server { - listen 8888; - root /Users/yeqing/projects/crawlab-frontend/dist; + listen 8080; + root /opt/dist; index index.html; location ~ .*\.(js|css)?$ { From e2c3923d6b120071794d0fab7709475062cef8b6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 21:39:43 +0800 Subject: [PATCH 60/80] updated frontend dockerfile --- frontend/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 554a3b8a..d6e4563b 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -9,6 +9,10 @@ ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION 8.12.0 ENV WORK_DIR /opt/crawlab/frontend +# install git curl +RUN apt-get update +RUN apt-get install git curl + # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ && . $NVM_DIR/nvm.sh \ From a2e49e1be52077370c8bb5f815e5dfe317f143ea Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 21:42:46 +0800 Subject: [PATCH 61/80] updated frontend dockerfile --- frontend/Dockerfile | 4 ++++ frontend/sources.list | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 frontend/sources.list diff --git a/frontend/Dockerfile b/frontend/Dockerfile index d6e4563b..81fa5591 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -4,6 +4,10 @@ FROM nginx:latest # source files ADD . /opt/crawlab/frontend +#用ubuntu国内源替换默认源 +RUN rm /etc/apt/sources.list +COPY sources.list /etc/apt/sources.list + # environment variables ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION 8.12.0 diff --git a/frontend/sources.list b/frontend/sources.list new file mode 100644 index 00000000..21751f3d --- /dev/null +++ b/frontend/sources.list @@ -0,0 +1,17 @@ +deb-src http://archive.ubuntu.com/ubuntu xenial main restricted #Added by software-properties +deb http://mirrors.aliyun.com/ubuntu/ xenial main restricted +deb-src http://mirrors.aliyun.com/ubuntu/ xenial main restricted multiverse universe #Added by software-properties +deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted +deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted multiverse universe #Added by software-properties +deb http://mirrors.aliyun.com/ubuntu/ xenial universe +deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe +deb http://mirrors.aliyun.com/ubuntu/ xenial multiverse +deb http://mirrors.aliyun.com/ubuntu/ xenial-updates multiverse +deb http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse +deb-src http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse #Added by software-properties +deb http://archive.canonical.com/ubuntu xenial partner +deb-src http://archive.canonical.com/ubuntu xenial partner +deb http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted +deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted multiverse universe #Added by software-properties +deb http://mirrors.aliyun.com/ubuntu/ xenial-security universe +deb http://mirrors.aliyun.com/ubuntu/ xenial-security multiverse From 32e82ac44459ccb1d476813a71b80a28d7b08aa1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 21:47:44 +0800 Subject: [PATCH 62/80] updated frontend dockerfile --- Dockerfile | 4 ++++ frontend/Dockerfile | 4 ---- frontend/sources.list | 17 ----------------- 3 files changed, 4 insertions(+), 21 deletions(-) delete mode 100644 frontend/sources.list diff --git a/Dockerfile b/Dockerfile index 2c18cc39..531d1d75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,10 @@ # images FROM python:3.6 +#用ubuntu国内源替换默认源 +RUN rm /etc/apt/sources.list +COPY sources.list /etc/apt/sources.list + # source files ADD . /opt/crawlab diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 81fa5591..d6e4563b 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -4,10 +4,6 @@ FROM nginx:latest # source files ADD . /opt/crawlab/frontend -#用ubuntu国内源替换默认源 -RUN rm /etc/apt/sources.list -COPY sources.list /etc/apt/sources.list - # environment variables ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION 8.12.0 diff --git a/frontend/sources.list b/frontend/sources.list deleted file mode 100644 index 21751f3d..00000000 --- a/frontend/sources.list +++ /dev/null @@ -1,17 +0,0 @@ -deb-src http://archive.ubuntu.com/ubuntu xenial main restricted #Added by software-properties -deb http://mirrors.aliyun.com/ubuntu/ xenial main restricted -deb-src http://mirrors.aliyun.com/ubuntu/ xenial main restricted multiverse universe #Added by software-properties -deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted -deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted multiverse universe #Added by software-properties -deb http://mirrors.aliyun.com/ubuntu/ xenial universe -deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe -deb http://mirrors.aliyun.com/ubuntu/ xenial multiverse -deb http://mirrors.aliyun.com/ubuntu/ xenial-updates multiverse -deb http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse -deb-src http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse #Added by software-properties -deb http://archive.canonical.com/ubuntu xenial partner -deb-src http://archive.canonical.com/ubuntu xenial partner -deb http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted -deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted multiverse universe #Added by software-properties -deb http://mirrors.aliyun.com/ubuntu/ xenial-security universe -deb http://mirrors.aliyun.com/ubuntu/ xenial-security multiverse From 2eeb4f79a22048866bc5238b2abc36a825e0c4c3 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 21:49:28 +0800 Subject: [PATCH 63/80] updated frontend dockerfile --- frontend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index d6e4563b..74daff44 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -11,7 +11,7 @@ ENV WORK_DIR /opt/crawlab/frontend # install git curl RUN apt-get update -RUN apt-get install git curl +RUN apt-get install -y git curl # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ From 3349e447e9f315efbda4403d3458c0ca90a234a1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 21:58:38 +0800 Subject: [PATCH 64/80] updated frontend dockerfile --- frontend/Dockerfile | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 74daff44..2c7237a8 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,26 +1,27 @@ # images -FROM nginx:latest +FROM node:8.12 # source files ADD . /opt/crawlab/frontend # environment variables -ENV NVM_DIR /usr/local/nvm -ENV NODE_VERSION 8.12.0 -ENV WORK_DIR /opt/crawlab/frontend +#ENV NVM_DIR /usr/local/nvm +#ENV NODE_VERSION 8.12.0 +#ENV WORK_DIR /opt/crawlab/frontend # install git curl RUN apt-get update -RUN apt-get install -y git curl +RUN apt-get install -y nginx +#RUN apt-get install -y git curl # install nvm -RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ - && . $NVM_DIR/nvm.sh \ - && nvm install v$NODE_VERSION \ - && nvm use v$NODE_VERSION \ - && nvm alias default v$NODE_VERSION -ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules -ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH +#RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ +# && . $NVM_DIR/nvm.sh \ +# && nvm install v$NODE_VERSION \ +# && nvm use v$NODE_VERSION \ +# && nvm alias default v$NODE_VERSION +#ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules +#ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH # install frontend RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org From a2bbadbb47a4be3cce1a6bd7ddc526ec383a885b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:02:47 +0800 Subject: [PATCH 65/80] updated frontend dockerfile --- frontend/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 2c7237a8..f49e691f 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -10,8 +10,7 @@ ADD . /opt/crawlab/frontend #ENV WORK_DIR /opt/crawlab/frontend # install git curl -RUN apt-get update -RUN apt-get install -y nginx +RUN apt-get update && apt-get install -y nginx #RUN apt-get install -y git curl # install nvm @@ -28,7 +27,6 @@ RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.taobao.org # nginx config & start frontend -RUN cp $WORK_DIR/conf/crawlab.conf /etc/nginx/conf.d -RUN service nginx reload +RUN cp $WORK_DIR/conf/crawlab.conf /etc/nginx/conf.d && service nginx reload CMD ["npm", "run", "build:prod"] From 60575e15421453a073f77af6cca505859c814f16 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:06:44 +0800 Subject: [PATCH 66/80] updated frontend dockerfile --- frontend/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index f49e691f..f84a3656 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -4,6 +4,13 @@ FROM node:8.12 # source files ADD . /opt/crawlab/frontend +#更新apt-get源 使用163的源 +RUN mv /etc/apt/sources.list /etc/apt/sources.list.bak && \ + echo "deb http://mirrors.163.com/debian/ jessie main non-free contrib" >/etc/apt/sources.list && \ + echo "deb http://mirrors.163.com/debian/ jessie-proposed-updates main non-free contrib" >>/etc/apt/sources.list && \ + echo "deb-src http://mirrors.163.com/debian/ jessie main non-free contrib" >>/etc/apt/sources.list && \ + echo "deb-src http://mirrors.163.com/debian/ jessie-proposed-updates main non-free contrib" >>/etc/apt/sources.list + # environment variables #ENV NVM_DIR /usr/local/nvm #ENV NODE_VERSION 8.12.0 From b91ddce5e8a71cbc3fe9bb2c399f31ce6cd82e06 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:10:41 +0800 Subject: [PATCH 67/80] updated frontend dockerfile --- frontend/Dockerfile | 7 ++----- frontend/sources.list | 8 ++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 frontend/sources.list diff --git a/frontend/Dockerfile b/frontend/Dockerfile index f84a3656..43bab2b0 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -5,11 +5,8 @@ FROM node:8.12 ADD . /opt/crawlab/frontend #更新apt-get源 使用163的源 -RUN mv /etc/apt/sources.list /etc/apt/sources.list.bak && \ - echo "deb http://mirrors.163.com/debian/ jessie main non-free contrib" >/etc/apt/sources.list && \ - echo "deb http://mirrors.163.com/debian/ jessie-proposed-updates main non-free contrib" >>/etc/apt/sources.list && \ - echo "deb-src http://mirrors.163.com/debian/ jessie main non-free contrib" >>/etc/apt/sources.list && \ - echo "deb-src http://mirrors.163.com/debian/ jessie-proposed-updates main non-free contrib" >>/etc/apt/sources.list +RUN mv /etc/apt/sources.list /etc/apt/sources.list.bak +COPY sources.list /etc/apt/sources.list # environment variables #ENV NVM_DIR /usr/local/nvm diff --git a/frontend/sources.list b/frontend/sources.list new file mode 100644 index 00000000..a9c2a4ff --- /dev/null +++ b/frontend/sources.list @@ -0,0 +1,8 @@ +deb http://mirrors.163.com/debian/ jessie main non-free contrib +deb http://mirrors.163.com/debian/ jessie-updates main non-free contrib +deb http://mirrors.163.com/debian/ jessie-backports main non-free contrib +deb-src http://mirrors.163.com/debian/ jessie main non-free contrib +deb-src http://mirrors.163.com/debian/ jessie-updates main non-free contrib +deb-src http://mirrors.163.com/debian/ jessie-backports main non-free contrib +deb http://mirrors.163.com/debian-security/ jessie/updates main non-free contrib +deb-src http://mirrors.163.com/debian-security/ jessie/updates main non-free contrib From a287e18ecc011a999d3574e25ca7d01009159233 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:14:43 +0800 Subject: [PATCH 68/80] updated frontend dockerfile --- frontend/sources.list | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/frontend/sources.list b/frontend/sources.list index a9c2a4ff..55420106 100644 --- a/frontend/sources.list +++ b/frontend/sources.list @@ -1,8 +1,6 @@ -deb http://mirrors.163.com/debian/ jessie main non-free contrib -deb http://mirrors.163.com/debian/ jessie-updates main non-free contrib -deb http://mirrors.163.com/debian/ jessie-backports main non-free contrib -deb-src http://mirrors.163.com/debian/ jessie main non-free contrib -deb-src http://mirrors.163.com/debian/ jessie-updates main non-free contrib -deb-src http://mirrors.163.com/debian/ jessie-backports main non-free contrib -deb http://mirrors.163.com/debian-security/ jessie/updates main non-free contrib -deb-src http://mirrors.163.com/debian-security/ jessie/updates main non-free contrib +deb http://mirrors.aliyun.com/debian wheezy main contrib non-free +deb-src http://mirrors.aliyun.com/debian wheezy main contrib non-free +deb http://mirrors.aliyun.com/debian wheezy-updates main contrib non-free +deb-src http://mirrors.aliyun.com/debian wheezy-updates main contrib non-free +deb http://mirrors.aliyun.com/debian-security wheezy/updates main contrib non-free +deb-src http://mirrors.aliyun.com/debian-security wheezy/updates main contrib non-free From 46370189bab827f61ce2c98861a10ab318f3e5c2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:18:03 +0800 Subject: [PATCH 69/80] updated frontend dockerfile --- frontend/sources.list | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/frontend/sources.list b/frontend/sources.list index 55420106..6ff85f91 100644 --- a/frontend/sources.list +++ b/frontend/sources.list @@ -1,6 +1,4 @@ -deb http://mirrors.aliyun.com/debian wheezy main contrib non-free -deb-src http://mirrors.aliyun.com/debian wheezy main contrib non-free -deb http://mirrors.aliyun.com/debian wheezy-updates main contrib non-free -deb-src http://mirrors.aliyun.com/debian wheezy-updates main contrib non-free -deb http://mirrors.aliyun.com/debian-security wheezy/updates main contrib non-free -deb-src http://mirrors.aliyun.com/debian-security wheezy/updates main contrib non-free +deb http://mirrors.aliyun.com/debian/ jessie main non-free contrib +deb http://mirrors.aliyun.com/debian/ jessie-proposed-updates main non-free contrib +deb-src http://mirrors.aliyun.com/debian/ jessie main non-free contrib +deb-src http://mirrors.aliyun.com/debian/ jessie-proposed-updates main non-free contrib From b0a6640f6b733ee6f9fd6e26f88325e82c33cc2c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:20:22 +0800 Subject: [PATCH 70/80] updated frontend dockerfile --- frontend/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 43bab2b0..bfcc0d49 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -5,8 +5,8 @@ FROM node:8.12 ADD . /opt/crawlab/frontend #更新apt-get源 使用163的源 -RUN mv /etc/apt/sources.list /etc/apt/sources.list.bak -COPY sources.list /etc/apt/sources.list +#RUN mv /etc/apt/sources.list /etc/apt/sources.list.bak +#COPY sources.list /etc/apt/sources.list # environment variables #ENV NVM_DIR /usr/local/nvm From cc0fefb077bd829dffe4c3559abf7eff5cc322c8 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:27:49 +0800 Subject: [PATCH 71/80] updated Dockerfile --- Dockerfile | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 531d1d75..965818a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,5 @@ # images -FROM python:3.6 - -#用ubuntu国内源替换默认源 -RUN rm /etc/apt/sources.list -COPY sources.list /etc/apt/sources.list +FROM ubuntu:latest # source files ADD . /opt/crawlab @@ -28,9 +24,9 @@ RUN apt-get install -y mongodb-org RUN mongod >> /var/log/mongod.log 2>&1 & # install python -#RUN apt-get install -y python python3 python3-pip -#RUN ln -s /usr/bin/pip3 /usr/local/bin/pip -#RUN ln -s /usr/bin/python3 /usr/local/bin/python +RUN apt-get install -y python python3 python3-pip +RUN ln -s /usr/bin/pip3 /usr/local/bin/pip +RUN ln -s /usr/bin/python3 /usr/local/bin/python #RUN git clone git://github.com/yyuu/pyenv.git ~/.pyenv #ENV PYENV_ROOT "$HOME/.pyenv" #ENV PATH "$PYENV_ROOT/bin:$PATH" From 00aab1a67c24e21df1d63cd981978c72d4090395 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:33:45 +0800 Subject: [PATCH 72/80] updated Dockerfile --- Dockerfile | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index 965818a0..04a66400 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,20 +13,22 @@ ENV NODE_VERSION 8.12.0 ENV WORK_DIR /opt/crawlab # install pkg -RUN apt-get update -RUN apt-get install -y curl git net-tools iputils-ping ntp gnupg2 +RUN apt-get update \ + && apt-get install -y curl git net-tools iputils-ping ntp gnupg2 nginx redis python python3 python3-pip \ + && service redis-server start \ + && cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d \ + && service nginx reload \ + && ln -s /usr/bin/pip3 /usr/local/bin/pip \ + && ln -s /usr/bin/python3 /usr/local/bin/python # install mongodb -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list -RUN apt-get update -RUN apt-get install -y mongodb-org -RUN mongod >> /var/log/mongod.log 2>&1 & +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ + && echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ + && apt-get update \ + && apt-get install -y mongodb-org \ + && mongod >> /var/log/mongod.log 2>&1 & # install python -RUN apt-get install -y python python3 python3-pip -RUN ln -s /usr/bin/pip3 /usr/local/bin/pip -RUN ln -s /usr/bin/python3 /usr/local/bin/python #RUN git clone git://github.com/yyuu/pyenv.git ~/.pyenv #ENV PYENV_ROOT "$HOME/.pyenv" #ENV PATH "$PYENV_ROOT/bin:$PATH" @@ -44,23 +46,14 @@ ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH # install frontend -RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org -RUN cd /opt/crawlab/frontend && yarn install --registry=https://registry.npm.taobao.org - -# install nginx -RUN apt-get install -y nginx - -# install redis -RUN apt-get install -y redis-server -RUN service redis-server start +RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org \ + && cd /opt/crawlab/frontend \ + && yarn install --registry=https://registry.npm.taobao.org # install backend -RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple -RUN pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple \ + && pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple -# nginx config & start frontend -RUN cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d -RUN service nginx reload # start backend EXPOSE 8080 From b8946cc32073e3ab5a1249276ece3be8b39ef853 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:45:25 +0800 Subject: [PATCH 73/80] updated Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 04a66400..470b5ae4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,8 +25,8 @@ RUN apt-get update \ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ && echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ && apt-get update \ - && apt-get install -y mongodb-org \ - && mongod >> /var/log/mongod.log 2>&1 & + && apt-get install -y mongodb-org +RUN mongod >> /var/log/mongod.log 2>&1 & # install python #RUN git clone git://github.com/yyuu/pyenv.git ~/.pyenv From f0b127c5f9f52217ac5823da0dbeefbaafee79c7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 22:45:58 +0800 Subject: [PATCH 74/80] added docker_init.sh --- docker_init.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 docker_init.sh diff --git a/docker_init.sh b/docker_init.sh new file mode 100755 index 00000000..e6d9d394 --- /dev/null +++ b/docker_init.sh @@ -0,0 +1,3 @@ +cd /opt/crawlab/frontend \ + && npm run build:prod \ + && python3 $WORK_DIR/manage.py \ No newline at end of file From bdeecea0896004db409fa42ca82141ee4029296c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 23:02:08 +0800 Subject: [PATCH 75/80] added docker_init.sh --- docker_init.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker_init.sh b/docker_init.sh index e6d9d394..a2758200 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -1,3 +1,4 @@ +#!/bin/sh cd /opt/crawlab/frontend \ && npm run build:prod \ - && python3 $WORK_DIR/manage.py \ No newline at end of file + && python3 $WORK_DIR/manage.py @* \ No newline at end of file From ad0d689eae1dee04a3107ecf7c84ebddc04f9ed0 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 23:15:13 +0800 Subject: [PATCH 76/80] updated Dockerfile --- docker_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_init.sh b/docker_init.sh index a2758200..6c255fbf 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -1,4 +1,4 @@ #!/bin/sh cd /opt/crawlab/frontend \ && npm run build:prod \ - && python3 $WORK_DIR/manage.py @* \ No newline at end of file + && python3 $WORK_DIR/manage.py $* \ No newline at end of file From 17b2901fa56b847379d04ba1b209f25d852f7352 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 10 Jun 2019 23:51:02 +0800 Subject: [PATCH 77/80] updated Dockerfile --- Dockerfile | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 470b5ae4..e5aca300 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,16 +25,9 @@ RUN apt-get update \ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 \ && echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ && apt-get update \ - && apt-get install -y mongodb-org -RUN mongod >> /var/log/mongod.log 2>&1 & - -# install python -#RUN git clone git://github.com/yyuu/pyenv.git ~/.pyenv -#ENV PYENV_ROOT "$HOME/.pyenv" -#ENV PATH "$PYENV_ROOT/bin:$PATH" -#RUN eval "$(pyenv init -)" -#RUN pyenv install 3.6.8 -#RUN pyenv local 3.6.8 + && apt-get install -y mongodb-org \ + && mkdir -p /data/db \ + && mongod --fork --logpath /var/log/mongod.log # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ @@ -54,7 +47,6 @@ RUN npm install -g yarn pm2 --registry=https://registry.npm.taobao.org \ RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple \ && pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - # start backend EXPOSE 8080 EXPOSE 8000 From 657029c18d55d1eff63982b55de743f36b53037a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 11 Jun 2019 00:26:42 +0800 Subject: [PATCH 78/80] updated Dockerfile --- Dockerfile | 7 +++---- docker_init.sh | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5aca300..ec67bd6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,8 @@ ENV WORK_DIR /opt/crawlab # install pkg RUN apt-get update \ && apt-get install -y curl git net-tools iputils-ping ntp gnupg2 nginx redis python python3 python3-pip \ - && service redis-server start \ + && apt-get clean \ && cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d \ - && service nginx reload \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python @@ -26,8 +25,8 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75 && echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list \ && apt-get update \ && apt-get install -y mongodb-org \ - && mkdir -p /data/db \ - && mongod --fork --logpath /var/log/mongod.log + && apt-get clean \ + && mkdir -p /data/db # install nvm RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ diff --git a/docker_init.sh b/docker_init.sh index 6c255fbf..4557b442 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -1,4 +1,7 @@ #!/bin/sh cd /opt/crawlab/frontend \ && npm run build:prod \ + && mongod --fork --logpath /var/log/mongod.log \ + && service nginx reload \ + && service redis-server start \ && python3 $WORK_DIR/manage.py $* \ No newline at end of file From 85baadc9513180771cd0b0418a46509269273419 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 11 Jun 2019 00:45:13 +0800 Subject: [PATCH 79/80] updated Dockerfile --- docker_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_init.sh b/docker_init.sh index 4557b442..956ef4e4 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -2,6 +2,6 @@ cd /opt/crawlab/frontend \ && npm run build:prod \ && mongod --fork --logpath /var/log/mongod.log \ - && service nginx reload \ + && service nginx start \ && service redis-server start \ && python3 $WORK_DIR/manage.py $* \ No newline at end of file From 6e215f2de2b4ea067f3172c5052a551d177eff24 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 11 Jun 2019 08:08:58 +0800 Subject: [PATCH 80/80] updated Dockerfile --- docker_init.sh | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docker_init.sh b/docker_init.sh index 956ef4e4..59208c8e 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -1,7 +1,17 @@ #!/bin/sh -cd /opt/crawlab/frontend \ - && npm run build:prod \ - && mongod --fork --logpath /var/log/mongod.log \ - && service nginx start \ - && service redis-server start \ - && python3 $WORK_DIR/manage.py $* \ No newline at end of file +case $1 in + master) + cd /opt/crawlab/frontend \ + && npm run build:prod \ + && service nginx start \ + && mongod --fork --logpath /var/log/mongod.log + redis-server >> /var/log/redis-server.log 2>&1 & + python $WORK_DIR/crawlab/flower.py >> /opt/crawlab/flower.log 2>&1 & + python $WORK_DIR/crawlab/worker.py >> /opt/crawlab/worker.log 2>&1 & + python $WORK_DIR/crawlab/app.py + ;; + worker) + python $WORK_DIR/crawlab/app.py >> /opt/crawlab/app.log 2>&1 & + python $WORK_DIR/crawlab/worker.py + ;; +esac \ No newline at end of file