From 47a494a0120ae9f5941e937c8e37e5855ae040f8 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 10 Feb 2020 17:11:10 +0800 Subject: [PATCH 1/2] updated etl --- backend/scripts/install-nodejs.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/backend/scripts/install-nodejs.sh b/backend/scripts/install-nodejs.sh index 4eeb8707..6fd8cfb0 100644 --- a/backend/scripts/install-nodejs.sh +++ b/backend/scripts/install-nodejs.sh @@ -17,14 +17,28 @@ ln -s $HOME/.nvm/versions/node/v8.12.0/bin/node /usr/local/bin/node export NODE_PATH=$HOME.nvm/versions/node/v8.12.0/lib/node_modules export PATH=$NODE_PATH:$PATH -# install apt dependencies -apt-get install -y wget -wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ - && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ - && apt-get update \ - && apt-get install -y google-chrome-unstable \ - --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* +# # install chromium +# # See https://crbug.com/795759 +# RUN apt-get update && apt-get install -yq libgconf-2-4 + +# # Install latest chrome dev package and fonts to support major +# # charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) +# # Note: this installs the necessary libs to make the bundled version +# # of Chromium that Puppeteer +# # installs, work. +# RUN apt-get update && apt-get install -y wget --no-install-recommends \ +# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ +# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ +# && apt-get update \ +# && apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ +# --no-install-recommends \ +# && rm -rf /var/lib/apt/lists/* \ +# && apt-get purge --auto-remove -y curl \ +# && rm -rf /src/*.deb + +# # It's a good idea to use dumb-init to help prevent zombie chrome processes. +# ADD https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 /usr/local/bin/dumb-init +# RUN chmod +x /usr/local/bin/dumb-init # install default dependencies export PUPPETEER_DOWNLOAD_HOST=https://npm.taobao.org/mirrors From 3e2bdb59f4241a4bd23e7bb67bdfb1b5ee0dfaa5 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 10 Feb 2020 17:41:14 +0800 Subject: [PATCH 2/2] updated install-nodejs.sh --- backend/scripts/install-nodejs.sh | 44 ++++++++++++++----------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/backend/scripts/install-nodejs.sh b/backend/scripts/install-nodejs.sh index 6fd8cfb0..2aa05fba 100644 --- a/backend/scripts/install-nodejs.sh +++ b/backend/scripts/install-nodejs.sh @@ -6,7 +6,7 @@ export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || pr [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm # install Node.js v8.12 -# export NVM_NODEJS_ORG_MIRROR=http://npm.taobao.org/mirrors/node +export NVM_NODEJS_ORG_MIRROR=http://npm.taobao.org/mirrors/node nvm install 8.12 # create soft links @@ -17,29 +17,25 @@ ln -s $HOME/.nvm/versions/node/v8.12.0/bin/node /usr/local/bin/node export NODE_PATH=$HOME.nvm/versions/node/v8.12.0/lib/node_modules export PATH=$NODE_PATH:$PATH -# # install chromium -# # See https://crbug.com/795759 -# RUN apt-get update && apt-get install -yq libgconf-2-4 +# install chromium +# See https://crbug.com/795759 +RUN apt-get update && apt-get install -yq libgconf-2-4 -# # Install latest chrome dev package and fonts to support major -# # charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) -# # Note: this installs the necessary libs to make the bundled version -# # of Chromium that Puppeteer -# # installs, work. -# RUN apt-get update && apt-get install -y wget --no-install-recommends \ -# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ -# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ -# && apt-get update \ -# && apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ -# --no-install-recommends \ -# && rm -rf /var/lib/apt/lists/* \ -# && apt-get purge --auto-remove -y curl \ -# && rm -rf /src/*.deb - -# # It's a good idea to use dumb-init to help prevent zombie chrome processes. -# ADD https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 /usr/local/bin/dumb-init -# RUN chmod +x /usr/local/bin/dumb-init +# Install latest chrome dev package and fonts to support major +# charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) +# Note: this installs the necessary libs to make the bundled version +# of Chromium that Puppeteer +# installs, work. +RUN apt-get update && apt-get install -y wget --no-install-recommends \ + && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ + && apt-get update \ + && apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get purge --auto-remove -y curl \ + && rm -rf /src/*.deb # install default dependencies -export PUPPETEER_DOWNLOAD_HOST=https://npm.taobao.org/mirrors -npm install puppeteer-chromium-resolver crawlab-sdk -g --unsafe-perm=true +PUPPETEER_DOWNLOAD_HOST=https://npm.taobao.org/mirrors +npm install puppeteer-chromium-resolver crawlab-sdk -g --unsafe-perm=true --registry=https://registry.npm.taobao.org