updated Dockerfile

This commit is contained in:
Marvin Zhang
2019-07-28 15:27:44 +08:00
parent 80449a8d39
commit f3e78423f8
15 changed files with 610 additions and 69 deletions

View File

@@ -1,4 +1,5 @@
.idea
logs
*.log
node_modules/
dist/
**/node_modules/

View File

@@ -1,45 +1,59 @@
FROM golang:1.12 AS backend-build
WORKDIR /go/src/app
COPY ./backend .
ENV GO111MODULE on
ENV GOPROXY https://mirrors.aliyun.com/goproxy/
RUN go install -v ./...
FROM node:8.16.0-alpine AS frontend-build
ADD ./frontend /app
WORKDIR /app
# install frontend
RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org
RUN npm run build:prod
# images
FROM ubuntu:latest
# source files
ADD . /opt/crawlab
ADD . /app
# set as non-interactive
ENV DEBIAN_FRONTEND noninteractive
# environment variables
ENV NVM_DIR /usr/local/nvm
ENV NODE_VERSION 8.12.0
ENV WORK_DIR /opt/crawlab
# install pkg
# install packages
RUN apt-get update \
&& apt-get install -y curl git net-tools iputils-ping ntp nginx python3 python3-pip \
&& apt-get clean \
&& cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip \
&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
&& ln -s /usr/bin/python3 /usr/local/bin/python
# install nvm
RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \
&& . $NVM_DIR/nvm.sh \
&& nvm install v$NODE_VERSION \
&& nvm use v$NODE_VERSION \
&& nvm alias default v$NODE_VERSION
ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules
ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
# install frontend
RUN npm install -g yarn \
&& cd /opt/crawlab/frontend \
&& yarn install
# install backend
RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple \
&& pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple
# copy backend files
COPY --from=backend-build /go/src/app .
COPY --from=backend-build /go/bin/crawlab /usr/local/bin
# install nginx
RUN apt-get -y install nginx
# copy frontend files
COPY --from=frontend-build /app/dist /app/dist
COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d
# working directory
WORKDIR /app/backend
# frontend port
EXPOSE 8080
# backend port
EXPOSE 8000
# start backend
EXPOSE 8080
EXPOSE 8000
WORKDIR /opt/crawlab
ENTRYPOINT ["/bin/sh", "/opt/crawlab/docker_init.sh"]
CMD ["/bin/sh", "/app/docker_init.sh"]

View File

@@ -1,7 +1,11 @@
api:
address: "localhost:8000"
mongo:
host: localhost
port: 27017
db: crawlab_test
username: ""
password: ""
redis:
network: tcp
address: "localhost:6379"
@@ -11,10 +15,10 @@ log:
server:
host: 0.0.0.0
port: 8000
master: "Y"
master: "N"
secret: "crawlab"
spider:
path: "/Users/yeqing/projects/crawlab/spiders"
path: "/app/spiders"
task:
workers: 4
other:

View File

@@ -21,8 +21,9 @@ type Node struct {
// 前端展示
IsMaster bool `json:"is_master"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTsUnix int64 `json:"update_ts_unix" bson:"update_ts_unix"`
}
func (n *Node) Save() error {
@@ -40,6 +41,7 @@ func (n *Node) Add() error {
defer s.Close()
n.Id = bson.NewObjectId()
n.UpdateTs = time.Now()
n.UpdateTsUnix = time.Now().Unix()
n.CreateTs = time.Now()
if err := c.Insert(&n); err != nil {
debug.PrintStack()

View File

@@ -16,10 +16,11 @@ import (
)
type Data struct {
Mac string `json:"mac"`
Ip string `json:"ip"`
Master bool `json:"master"`
UpdateTs time.Time `json:"update_ts"`
Mac string `json:"mac"`
Ip string `json:"ip"`
Master bool `json:"master"`
UpdateTs time.Time `json:"update_ts"`
UpdateTsUnix int64 `json:"update_ts_unix"`
}
type NodeMessage struct {
@@ -193,9 +194,8 @@ func UpdateNodeStatus() {
}
// 如果记录的更新时间超过60秒该节点被认为离线
if time.Now().Sub(data.UpdateTs) > 60*time.Second {
if time.Now().Unix()-data.UpdateTsUnix > 60 {
// 在Redis中删除该节点
if err := database.RedisClient.HDel("nodes", data.Mac); err != nil {
log.Errorf(err.Error())
return
@@ -284,10 +284,11 @@ func UpdateNodeData() {
// 构造节点数据
data := Data{
Mac: mac,
Ip: ip,
Master: IsMaster(),
UpdateTs: time.Now(),
Mac: mac,
Ip: ip,
Master: IsMaster(),
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
}
// 注册节点到Redis

View File

@@ -124,10 +124,15 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
cmd.Stdout = fLog
cmd.Stderr = fLog
// 添加环境变量
// 添加默认环境变量
cmd.Env = append(cmd.Env, "CRAWLAB_TASK_ID="+t.Id)
cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+s.Col)
// 添加任务环境变量
for _, env := range s.Envs {
cmd.Env = append(cmd.Env, env.Name + "=" + env.Value)
}
// 起一个goroutine来监控进程
ch := TaskExecChanMap.ChanBlocked(t.Id)
go func() {
@@ -393,7 +398,7 @@ func GetTaskLog(id string) (logStr string, err error) {
}
logStr = ""
if IsMaster() {
if IsMasterNode(task.NodeId.Hex()) {
// 若为主节点,获取本机日志
logBytes, err := GetLocalLog(task.LogPath)
logStr = string(logBytes)

View File

@@ -1,16 +1,10 @@
#!/bin/sh
case $1 in
master)
cd $WORK_DIR/frontend \
&& npm run build:prod \
&& service nginx start
python $WORK_DIR/crawlab/flower.py >> /opt/crawlab/flower.log 2>&1 &
python $WORK_DIR/crawlab/worker.py >> /opt/crawlab/worker.log 2>&1 &
cd $WORK_DIR/crawlab \
&& gunicorn --log-level=DEBUG -b 0.0.0.0 -w 8 app:app
;;
worker)
python $WORK_DIR/crawlab/app.py >> /opt/crawlab/app.log 2>&1 &
python $WORK_DIR/crawlab/worker.py
;;
esac
# replace default api path to new one
jspath=`ls /app/dist/js/app.*.js`
cat ${jspath} | sed "s/localhost:8000/${CRAWLAB_API_ADDRESS}/g" > ${jspath}
# start nginx
service nginx start
crawlab

View File

@@ -0,0 +1,75 @@
<template>
<el-scrollbar wrap-class="scrollbar-wrapper">
<div class="sidebar-logo" :class="isCollapse ? 'collapsed' : ''">
<span>C</span><span v-show="!isCollapse">rawlab</span>
</div>
<el-menu
:show-timeout="200"
:default-active="routeLevel1"
:collapse="isCollapse"
:background-color="variables.menuBg"
:text-color="variables.menuText"
:active-text-color="variables.menuActiveText"
mode="vertical"
>
<sidebar-item v-for="route in routes" :key="route.path" :item="route" :base-path="route.path"/>
</el-menu>
</el-scrollbar>
</template>
<script>
import { mapState, mapGetters } from 'vuex'
import variables from '@/styles/variables.scss'
import SidebarItem from './SidebarItem'
export default {
components: { SidebarItem },
computed: {
...mapState('user', [
'adminPaths'
]),
...mapGetters([
'sidebar'
]),
routeLevel1 () {
let pathArray = this.$route.path.split('/')
return `/${pathArray[1]}`
},
routes () {
return this.$router.options.routes.filter(d => {
const role = this.$store.getters['user/userInfo'].role
if (role === 'admin') return true
return !this.adminPaths.includes(d.path)
})
},
variables () {
return variables
},
isCollapse () {
return !this.sidebar.opened
}
}
}
</script>
<style>
#app .sidebar-container .el-menu {
height: calc(100% - 50px);
}
.sidebar-container .sidebar-logo {
height: 50px;
display: flex;
align-items: center;
padding-left: 20px;
color: #fff;
background: rgb(48, 65, 86);
font-size: 28px;
font-weight: 600;
font-family: "Verdana", serif;
}
.sidebar-container .sidebar-logo.collapsed {
padding-left: 7px;
}
</style>

View File

@@ -0,0 +1,411 @@
<template>
<div class="login-container">
<canvas id="canvas"></canvas>
<el-form ref="loginForm" :model="loginForm" :rules="loginRules" class="login-form" auto-complete="on"
label-position="left">
<h3 class="title">
CRAWLAB
</h3>
<el-form-item prop="username" style="margin-bottom: 28px;">
<el-input
v-model="loginForm.username"
name="username"
type="text"
auto-complete="on"
:placeholder="$t('Username')"
/>
</el-form-item>
<el-form-item prop="password" style="margin-bottom: 28px;">
<el-input
:type="pwdType"
v-model="loginForm.password"
name="password"
auto-complete="on"
:placeholder="$t('Password')"
@keyup.enter.native="onKeyEnter"/>
</el-form-item>
<el-form-item v-if="isSignUp" prop="confirmPassword" style="margin-bottom: 28px;">
<el-input
:type="pwdType"
v-model="loginForm.confirmPassword"
name="password"
auto-complete="on"
:placeholder="$t('Confirm Password')"
@keyup.enter.native="onKeyEnter"
/>
</el-form-item>
<el-form-item style="border: none">
<el-button v-if="isSignUp" :loading="loading" type="primary" style="width:100%;"
@click.native.prevent="handleSignup">
{{$t('Sign up')}}
</el-button>
<el-button v-if="!isSignUp" :loading="loading" type="primary" style="width:100%;"
@click.native.prevent="handleLogin">
{{$t('Sign in')}}
</el-button>
</el-form-item>
<div class="alternatives">
<div class="left">
<span v-if="!isSignUp" class="forgot-password">{{$t('Forgot Password')}}</span>
</div>
<div class="right">
<span v-if="isSignUp">{{$t('Has Account')}}, </span>
<span v-if="isSignUp" class="sign-in" @click="$router.push('/login')">{{$t('Sign-in')}} ></span>
<span v-if="!isSignUp">{{$t('New to Crawlab')}}, </span>
<span v-if="!isSignUp" class="sign-up" @click="$router.push('/signup')">{{$t('Sign-up')}} ></span>
</div>
</div>
<div class="tips">
<span>{{$t('Initial Username/Password')}}: admin/admin</span>
<a href="https://github.com/tikazyq/crawlab" target="_blank" style="float:right">
<img src="https://img.shields.io/badge/github-crawlab-blue">
</a>
</div>
</el-form>
</div>
</template>
<script>
import { isValidUsername } from '../../utils/validate'
export default {
name: 'Login',
data () {
const validateUsername = (rule, value, callback) => {
if (!isValidUsername(value)) {
callback(new Error(this.$t('Please enter the correct username')))
} else {
callback()
}
}
const validatePass = (rule, value, callback) => {
if (value.length < 5) {
callback(new Error(this.$t('Password length should be no shorter than 5')))
} else {
callback()
}
}
const validateConfirmPass = (rule, value, callback) => {
if (!this.isSignUp) return callback()
if (value !== this.loginForm.password) {
callback(new Error(this.$t('Two passwords must be the same')))
} else {
callback()
}
}
return {
loginForm: {
username: '',
password: '',
confirmPassword: ''
},
loginRules: {
username: [{ required: true, trigger: 'blur', validator: validateUsername }],
password: [{ required: true, trigger: 'blur', validator: validatePass }],
confirmPassword: [{ required: true, trigger: 'blur', validator: validateConfirmPass }]
},
loading: false,
pwdType: 'password'
}
},
computed: {
isSignUp () {
return this.$route.path === '/signup'
},
redirect () {
return this.$route.query.redirect
}
},
methods: {
handleLogin () {
this.$refs.loginForm.validate(valid => {
if (valid) {
this.loading = true
this.$store.dispatch('user/login', this.loginForm).then(() => {
this.loading = false
this.$router.push({ path: this.redirect || '/' })
this.$store.dispatch('user/getInfo')
}).catch(() => {
this.$message.error(this.$t('Error when logging in (Please check username and password)'))
this.loading = false
})
}
})
},
handleSignup () {
this.$refs.loginForm.validate(valid => {
if (valid) {
this.loading = true
this.$store.dispatch('user/register', this.loginForm).then(() => {
this.handleLogin()
this.loading = false
}).catch(err => {
this.$message.error(this.$t(err))
this.loading = false
})
}
})
},
onKeyEnter () {
const func = this.isSignUp ? this.handleSignup : this.handleLogin
func()
}
},
mounted () {
initCanvas()
}
}
const initCanvas = () => {
var canvas = document.getElementById('canvas')
var ctx = canvas.getContext('2d')
resize()
window.onresize = resize
function resize () {
canvas.width = window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth
canvas.height = window.innerHeight || document.documentElement.clientHeight || document.body.clientHeight
}
var RAF = (function () {
return window.requestAnimationFrame || window.webkitRequestAnimationFrame || window.mozRequestAnimationFrame || window.oRequestAnimationFrame || window.msRequestAnimationFrame || function (callback) {
window.setTimeout(callback, 1000 / 60)
}
})()
// 鼠标活动时,获取鼠标坐标
var warea = { x: null, y: null, max: 20000 }
// window.onmousemove = function (e) {
// e = e || window.event
//
// warea.x = e.clientX
// warea.y = e.clientY
// }
// window.onmouseout = function (e) {
// warea.x = null
// warea.y = null
// }
// 添加粒子
// xy为粒子坐标xa, ya为粒子xy轴加速度max为连线的最大距离
var dots = []
for (var i = 0; i < 300; i++) {
var x = Math.random() * canvas.width
var y = Math.random() * canvas.height
var xa = Math.random() * 2 - 1
var ya = Math.random() * 2 - 1
dots.push({
x: x,
y: y,
xa: xa,
ya: ya,
max: 6000
})
}
// 延迟100秒开始执行动画如果立即执行有时位置计算会出错
setTimeout(function () {
animate()
}, 100)
// 每一帧循环的逻辑
function animate () {
ctx.clearRect(0, 0, canvas.width, canvas.height)
// 将鼠标坐标添加进去,产生一个用于比对距离的点数组
var ndots = [warea].concat(dots)
dots.forEach(function (dot) {
// 粒子位移
dot.x += dot.xa
dot.y += dot.ya
// 遇到边界将加速度反向
dot.xa *= (dot.x > canvas.width || dot.x < 0) ? -1 : 1
dot.ya *= (dot.y > canvas.height || dot.y < 0) ? -1 : 1
// 绘制点
ctx.fillRect(dot.x - 0.5, dot.y - 0.5, 1, 1)
// 循环比对粒子间的距离
for (var i = 0; i < ndots.length; i++) {
var d2 = ndots[i]
if (dot === d2 || d2.x === null || d2.y === null) continue
var xc = dot.x - d2.x
var yc = dot.y - d2.y
// 两个粒子之间的距离
var dis = xc * xc + yc * yc
// 距离比
var ratio
// 如果两个粒子之间的距离小于粒子对象的max值则在两个粒子间画线
if (dis < d2.max) {
// 如果是鼠标,则让粒子向鼠标的位置移动
if (d2 === warea && dis > (d2.max / 2)) {
dot.x -= xc * 0.03
dot.y -= yc * 0.03
}
// 计算距离比
ratio = (d2.max - dis) / d2.max
// 画线
ctx.beginPath()
ctx.lineWidth = ratio / 2
// 线条颜色
ctx.strokeStyle = 'rgba(64,158,255,' + (ratio + 0.1) + ')'
ctx.moveTo(dot.x, dot.y)
ctx.lineTo(d2.x, d2.y)
ctx.stroke()
}
}
// 将已经计算过的粒子从数组中删除
ndots.splice(ndots.indexOf(dot), 1)
})
RAF(animate)
}
}
</script>
<style rel="stylesheet/scss" lang="scss">
$bg: #2d3a4b;
$light_gray: #eee;
/* reset element-ui css */
.login-container {
.el-input {
display: inline-block;
width: calc(100% - 44px);
margin-left: 22px;
input {
background: transparent;
border: 0;
-webkit-appearance: none;
border-radius: 0;
padding: 12px 5px 12px 15px;
color: #666;
height: 44px;
line-height: 44px;
}
}
.el-form-item {
border: 1px solid #ddd;
background: #fff;
border-radius: 22px;
color: #454545;
height: 44px;
/*margin-bottom: 28px;*/
.el-form-item__content {
line-height: 44px;
}
}
.el-button {
height: 44px;
border-radius: 22px;
}
#canvas {
position: fixed;
top: 0;
left: 0;
}
}
</style>
<style rel="stylesheet/scss" lang="scss" scoped>
$bg: transparent;
$dark_gray: #889aa4;
$light_gray: #aaa;
.login-container {
position: fixed;
height: 100%;
width: 100%;
background-color: $bg;
.login-form {
background: transparent;
position: absolute;
left: 0;
right: 0;
width: 480px;
max-width: 100%;
padding: 35px 35px 15px 35px;
margin: 120px auto;
}
.tips {
font-size: 14px;
color: #666;
margin-bottom: 10px;
background: transparent;
span {
&:first-of-type {
margin-right: 22px;
}
}
}
.svg-container {
padding: 6px 5px 6px 15px;
color: $dark_gray;
vertical-align: middle;
width: 30px;
display: inline-block;
}
.title {
font-family: "Verdana", serif;
font-style: italic;
font-weight: 600;
font-size: 32px;
/*color: ;*/
margin: 0px auto 20px auto;
text-align: center;
}
.show-pwd {
position: absolute;
right: 10px;
top: 7px;
font-size: 16px;
color: $dark_gray;
cursor: pointer;
user-select: none;
}
.alternatives {
border-bottom: 1px solid #ccc;
display: flex;
justify-content: space-between;
font-size: 14px;
color: #666;
font-weight: 400;
margin-bottom: 10px;
padding-bottom: 10px;
.forgot-password {
cursor: pointer;
}
.sign-in,
.sign-up {
cursor: pointer;
color: #409EFF;
font-weight: 600;
}
}
}
</style>

1
frontend/.dockerignore Normal file
View File

@@ -0,0 +1 @@
node_modules/

View File

@@ -1,2 +1,2 @@
NODE_ENV='production'
VUE_APP_BASE_URL='http://114.67.75.98:8000/api'
VUE_APP_BASE_URL='http://localhost:8000/api'

23
frontend/Dockerfile Normal file
View File

@@ -0,0 +1,23 @@
FROM node:8.16.0-alpine AS frontend-build
ADD . /app
WORKDIR /app
# install frontend
RUN npm install -g yarn \
&& yarn install --registry=https://registry.npm.taobao.org
RUN npm run build:prod
FROM alpine
#RUN apk update
RUN apk add nginx
COPY --from=frontend-build /app/dist /app/dist
COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d
#RUN nginx -s start
#COPY ./dist /usr/share/nginx/html
#EXPOSE 80
#EXPOSE 8080

View File

@@ -0,0 +1,5 @@
server {
listen 8080;
root /app/dist;
index index.html;
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

View File

@@ -4,7 +4,7 @@
<breadcrumb class="breadcrumb"/>
<el-dropdown class="avatar-container" trigger="click">
<span class="el-dropdown-link">
{{$t('User')}}
{{username}}
<i class="el-icon-arrow-down el-icon--right"></i>
</span>
<el-dropdown-menu slot="dropdown" class="user-dropdown">
@@ -19,12 +19,12 @@
<i class="el-icon-arrow-down el-icon--right"></i>
</span>
<el-dropdown-menu slot="dropdown">
<el-dropdown-item @click.native="setLang('en')">
<span>English</span>
</el-dropdown-item>
<el-dropdown-item @click.native="setLang('zh')">
<span>中文</span>
</el-dropdown-item>
<el-dropdown-item @click.native="setLang('en')">
<span>English</span>
</el-dropdown-item>
</el-dropdown-menu>
</el-dropdown>
</div>
@@ -44,7 +44,12 @@ export default {
...mapGetters([
'sidebar',
'avatar'
])
]),
username () {
if (!this.$store.getters['user/userInfo']) return this.$t('User')
if (!this.$store.getters['user/userInfo'].username) return this.$t('User')
return this.$store.getters['user/userInfo'].username
}
},
methods: {
toggleSideBar () {