git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-23 17:31:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #617 from crawlab-team/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2020-03-05 11:02:45 +08:00
committed by GitHub
parent f29b7eefb7 f0382ab768
commit 63343fe352
31 changed files with 325 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/dockerpush.yml vendored
View File

@@ -6,6 +6,7 @@ on:
branches:
- master
- release
- develop
# Publish `v1.2.3` tags as releases.
tags:
@@ -60,3 +61,6 @@ jobs:
if [ "$VERSION" == "release" ]; then
curl ${{ secrets.JENKINS_RELEASE_URL }}
fi
if [ "$VERSION" == "develop" ]; then
curl ${{ secrets.JENKINS_DEVELOP_URL }}
fi

4
Dockerfile
View File

@@ -51,7 +51,9 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server
# copy frontend files
COPY --from=frontend-build /app/dist /app/dist
COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d
# copy nginx config files
COPY ./nginx/crawlab.conf /etc/nginx/conf.d
# working directory
WORKDIR /app/backend

4
Dockerfile.local
View File

@@ -49,7 +49,9 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server
# copy frontend files
COPY --from=frontend-build /app/dist /app/dist
COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d
# copy nginx config files
COPY ./nginx/crawlab.conf /etc/nginx/conf.d
# working directory
WORKDIR /app/backend

2
backend/conf/config.yml
View File

@@ -23,7 +23,7 @@ server:
master: "Y"
secret: "crawlab"
register:
# mac地址 或者 ip地址，如果是ip，则需要手动指定IP
# mac地址/ip地址/hostname, 如果是ip，则需要手动指定IP
type: "mac"
ip: ""
lang: # 安装语言环境, Y 为安装，N 为不安装，只对 Docker 有效

7
backend/constants/register.go Normal file
View File

@@ -0,0 +1,7 @@
package constants
const (
RegisterTypeMac = "mac"
RegisterTypeIp = "ip"
RegisterTypeHostname = "hostname"
)

21
backend/model/node.go
View File

@@ -20,6 +20,7 @@ type Node struct {
Ip string `json:"ip" bson:"ip"`
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
// 用于唯一标识节点，可能是mac地址，可能是ip地址
Key string `json:"key" bson:"key"`
@@ -42,6 +43,7 @@ func IsMaster() bool {
}
// 获取本机节点
// TODO: 这里职责不单一，需要重构
func GetCurrentNode() (Node, error) {
// 获得注册的key值
key, err := register.GetRegister().GetKey()
@@ -67,7 +69,7 @@ func GetCurrentNode() (Node, error) {
//只在master节点运行的时候才检测master节点的信息是否存在
if IsMaster() && err == mgo.ErrNotFound {
// 获取本机信息
ip, mac, key, err := GetNodeBaseInfo()
ip, mac, hostname, key, err := GetNodeBaseInfo()
if err != nil {
debug.PrintStack()
return node, err
@@ -80,6 +82,7 @@ func GetCurrentNode() (Node, error) {
Ip: ip,
Name: ip,
Mac: mac,
Hostname: hostname,
IsMaster: true,
}
if err := node.Add(); err != nil {
@@ -239,25 +242,31 @@ func GetNodeCount(query interface{}) (int, error) {
}
// 节点基本信息
func GetNodeBaseInfo() (ip string, mac string, key string, error error) {
func GetNodeBaseInfo() (ip string, mac string, hostname string, key string, error error) {
ip, err := register.GetRegister().GetIp()
if err != nil {
debug.PrintStack()
return "", "", "", err
return "", "", "", "", err
}
mac, err = register.GetRegister().GetMac()
if err != nil {
debug.PrintStack()
return "", "", "", err
return "", "", "", "", err
}
hostname, err = register.GetRegister().GetHostname()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
key, err = register.GetRegister().GetKey()
if err != nil {
debug.PrintStack()
return "", "", "", err
return "", "", "", "", err
}
return ip, mac, key, nil
return ip, mac, key, hostname, nil
}
// 根据redis的key值，重置node节点为offline

2
backend/scripts/install-java.sh Normal file → Executable file
View File

@@ -4,7 +4,7 @@
touch /tmp/install-java.lock
# install java
apt-get install -y default-jdk
apt-get update && apt-get install -y default-jdk --fix-missing
ln -s /usr/bin/java /usr/local/bin/java
# unlock

11
backend/services/node.go
View File

@@ -23,6 +23,7 @@ type Data struct {
Key string `json:"key"`
Mac string `json:"mac"`
Ip string `json:"ip"`
Hostname string `json:"hostname"`
Master bool `json:"master"`
UpdateTs time.Time `json:"update_ts"`
UpdateTsUnix int64 `json:"update_ts_unix"`
@@ -161,6 +162,14 @@ func UpdateNodeData() {
log.Errorf(err.Error())
return
}
// 获取Hostname
hostname, err := register.GetRegister().GetHostname()
if err != nil {
log.Errorf(err.Error())
return
}
// 获取redis的key
key, err := register.GetRegister().GetKey()
if err != nil {
@@ -174,6 +183,7 @@ func UpdateNodeData() {
Key: key,
Mac: mac,
Ip: ip,
Hostname: hostname,
Master: model.IsMaster(),
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
@@ -191,7 +201,6 @@ func UpdateNodeData() {
log.Errorf(err.Error())
return
}
}
func MasterNodeCallback(message redis.Message) (err error) {

60
backend/services/register/register.go
View File

@@ -1,9 +1,13 @@
package register
import (
"bytes"
"crawlab/constants"
"fmt"
"github.com/apex/log"
"github.com/spf13/viper"
"net"
"os/exec"
"reflect"
"runtime/debug"
"sync"
@@ -18,6 +22,8 @@ type Register interface {
GetIp() (string, error)
// 注册节点的mac地址
GetMac() (string, error)
// 注册节点的Hostname
GetHostname() (string, error)
}
// ===================== mac 地址注册 =====================
@@ -39,6 +45,10 @@ func (mac *MacRegister) GetIp() (string, error) {
return getIp()
}
func (mac *MacRegister) GetHostname() (string, error) {
return getHostname()
}
// ===================== ip 地址注册 =====================
type IpRegister struct {
Ip string
@@ -60,6 +70,33 @@ func (ip *IpRegister) GetMac() (string, error) {
return getMac()
}
func (ip *IpRegister) GetHostname() (string, error) {
return getHostname()
}
// ===================== mac 地址注册 =====================
type HostnameRegister struct{}
func (h *HostnameRegister) GetType() string {
return "mac"
}
func (h *HostnameRegister) GetKey() (string, error) {
return h.GetHostname()
}
func (h *HostnameRegister) GetMac() (string, error) {
return getMac()
}
func (h *HostnameRegister) GetIp() (string, error) {
return getIp()
}
func (h *HostnameRegister) GetHostname() (string, error) {
return getHostname()
}
// ===================== 公共方法 =====================
// 获取本机的IP地址
// TODO: 考虑多个IP地址的情况
@@ -94,6 +131,23 @@ func getMac() (string, error) {
return "", nil
}
func getHostname() (string, error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("hostname")
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
log.Errorf(fmt.Sprintf("error: %s", stderr.String()))
debug.PrintStack()
return "", err
}
return stdout.String(), nil
}
// ===================== 获得注册简单工厂 =====================
var register Register
@@ -108,9 +162,9 @@ func GetRegister() Register {
}
registerType := viper.GetString("server.register.type")
if registerType == "mac" {
if registerType == constants.RegisterTypeMac {
register = &MacRegister{}
} else {
} else if registerType == constants.RegisterTypeIp {
ip := viper.GetString("server.register.ip")
if ip == "" {
log.Error("server.register.ip is empty")
@@ -120,6 +174,8 @@ func GetRegister() Register {
register = &IpRegister{
Ip: ip,
}
} else if registerType == constants.RegisterTypeHostname {
register = &HostnameRegister{}
}
log.Info("register type is :" + reflect.TypeOf(register).String())

2
backend/services/system.go
View File

@@ -67,7 +67,7 @@ func GetLangList(nodeId string) []entity.Lang {
list := []entity.Lang{
{Name: "Python", ExecutableName: "python", ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, DepExecutablePath: "/usr/local/bin/pip"},
{Name: "Node.js", ExecutableName: "node", ExecutablePaths: []string{"/usr/bin/node", "/usr/local/bin/node"}, DepExecutablePath: "/usr/local/bin/npm"},
{Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}},
//{Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}},
}
for i, lang := range list {
list[i].Installed = IsInstalledLang(nodeId, lang)

5
crawlab.conf
View File

@@ -1,5 +0,0 @@
server {
listen 8080;
root /opt/crawlab/frontend/dist;
index index.html;
}

50
devops/develop/crawlab-master.yaml Normal file
View File

@@ -0,0 +1,50 @@
apiVersion: v1
kind: Service
metadata:
name: crawlab
namespace: crawlab-develop
spec:
ports:
- port: 8080
targetPort: 8080
nodePort: 30108
selector:
app: crawlab-master
type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: crawlab-master
namespace: crawlab-develop
spec:
strategy:
type: Recreate
selector:
matchLabels:
app: crawlab-master
template:
metadata:
labels:
app: crawlab-master
spec:
containers:
- image: tikazyq/crawlab:develop
imagePullPolicy: Always
name: crawlab
env:
- name: CRAWLAB_SERVER_MASTER
value: "Y"
- name: CRAWLAB_MONGO_HOST
value: "mongo"
- name: CRAWLAB_REDIS_ADDRESS
value: "redis"
- name: CRAWLAB_SETTING_ALLOWREGISTER
value: "Y"
- name: CRAWLAB_SERVER_LANG_NODE
value: "Y"
- name: CRAWLAB_SERVER_LANG_JAVA
value: "Y"
ports:
- containerPort: 8080
name: crawlab

33
devops/develop/crawlab-worker.yaml Normal file
View File

@@ -0,0 +1,33 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: crawlab-worker
namespace: crawlab-develop
spec:
replicas: 2
strategy:
type: Recreate
selector:
matchLabels:
app: crawlab-worker
template:
metadata:
labels:
app: crawlab-worker
spec:
containers:
- image: tikazyq/crawlab:develop
imagePullPolicy: Always
name: crawlab
env:
- name: CRAWLAB_SERVER_MASTER
value: "N"
- name: CRAWLAB_MONGO_HOST
value: "mongo"
- name: CRAWLAB_REDIS_ADDRESS
value: "redis"
- name: CRAWLAB_SERVER_LANG_NODE
value: "Y"
- name: CRAWLAB_SERVER_LANG_JAVA
value: "Y"

28
devops/develop/mongo-pv.yaml Normal file
View File

@@ -0,0 +1,28 @@
apiVersion: v1
kind: PersistentVolume
metadata:
name: mongo-pv-volume-develop
namespace: crawlab-develop
labels:
type: local
spec:
storageClassName: manual
capacity:
storage: 2Gi
accessModes:
- ReadWriteOnce
hostPath:
path: "/data/crawlab-develop/mongodb/data"
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mongo-pv-claim-develop
namespace: crawlab-develop
spec:
storageClassName: manual
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi

41
devops/develop/mongo.yaml Normal file
View File

@@ -0,0 +1,41 @@
apiVersion: v1
kind: Service
metadata:
name: mongo
namespace: crawlab-develop
spec:
ports:
- port: 27017
selector:
app: mongo
clusterIP: None
---
apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2
kind: Deployment
metadata:
name: mongo
namespace: crawlab-develop
spec:
selector:
matchLabels:
app: mongo
strategy:
type: Recreate
template:
metadata:
labels:
app: mongo
spec:
containers:
- image: mongo:4
name: mongo
ports:
- containerPort: 27017
name: mongo
volumeMounts:
- name: mongo-persistent-storage
mountPath: /data/db
volumes:
- name: mongo-persistent-storage
persistentVolumeClaim:
claimName: mongo-pv-claim-develop

4
devops/develop/ns.yaml Normal file
View File

@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: crawlab-develop

34
devops/develop/redis.yaml Normal file
View File

@@ -0,0 +1,34 @@
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: crawlab-develop
spec:
ports:
- port: 6379
selector:
app: redis
clusterIP: None
---
apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2
kind: Deployment
metadata:
name: redis
namespace: crawlab-develop
spec:
selector:
matchLabels:
app: redis
strategy:
type: Recreate
template:
metadata:
labels:
app: redis
spec:
containers:
- image: redis
name: redis
ports:
- containerPort: 6379
name: redis

5
devops/release/crawlab-master.yaml
View File

@@ -30,6 +30,7 @@ spec:
spec:
containers:
- image: tikazyq/crawlab:release
imagePullPolicy: Always
name: crawlab
env:
- name: CRAWLAB_SERVER_MASTER
@@ -40,6 +41,10 @@ spec:
value: "redis"
- name: CRAWLAB_SETTING_ALLOWREGISTER
value: "Y"
- name: CRAWLAB_SERVER_LANG_NODE
value: "Y"
- name: CRAWLAB_SERVER_LANG_JAVA
value: "Y"
ports:
- containerPort: 8080
name: crawlab

7
devops/release/crawlab-worker.yaml
View File

@@ -17,6 +17,7 @@ spec:
spec:
containers:
- image: tikazyq/crawlab:release
imagePullPolicy: Always
name: crawlab
env:
- name: CRAWLAB_SERVER_MASTER
@@ -24,4 +25,8 @@ spec:
- name: CRAWLAB_MONGO_HOST
value: "mongo"
- name: CRAWLAB_REDIS_ADDRESS
value: "redis"
value: "redis"
- name: CRAWLAB_SERVER_LANG_NODE
value: "Y"
- name: CRAWLAB_SERVER_LANG_JAVA
value: "Y"

4
docker_init.sh
View File

@@ -26,14 +26,14 @@ service nginx start
if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ];
then
echo "installing node.js"
/bin/sh /app/backend/scripts/install-nodejs.sh &
/bin/sh /app/backend/scripts/install-nodejs.sh >> /var/log/install-nodejs.sh.log 2>&1 &
fi
# install languages: Java
if [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ];
then
echo "installing java"
/bin/sh /app/backend/scripts/install-java.sh &
/bin/sh /app/backend/scripts/install-java.sh >> /var/log/install-java.sh.log 2>&1 &
fi
# generate ssh

1
frontend/src/utils/tour.js
View File

@@ -21,6 +21,7 @@ export default {
startTour: (vm, tourName) => {
if (localStorage.getItem('enableTutorial') === '0') return
vm.$tours[tourName].start()
vm.$st.sendEv('教程', '开始', tourName)
},
finishTour: (tourName) => {
let data

1
frontend/src/views/node/NodeDetail.vue
View File

@@ -125,7 +125,6 @@ export default {
mounted () {
if (!this.$utils.tour.isFinishedTour('node-detail')) {
this.$utils.tour.startTour(this, 'node-detail')
this.$st.sendEv('教程', '开始', 'node-detail')
}
}
}

2
frontend/src/views/schedule/ScheduleList.vue
View File

@@ -468,7 +468,6 @@ export default {
if (!this.$utils.tour.isFinishedTour('schedule-list-add')) {
setTimeout(() => {
this.$utils.tour.startTour(this, 'schedule-list-add')
this.$st.sendEv('教程', '开始', 'schedule-list-add')
}, 500)
}
},
@@ -618,7 +617,6 @@ export default {
if (!this.isDisabledSpiderSchedule) {
if (!this.$utils.tour.isFinishedTour('schedule-list')) {
this.$utils.tour.startTour(this, 'schedule-list')
this.$st.sendEv('教程', '开始', 'schedule-list')
}
}
}

1
frontend/src/views/setting/Setting.vue
View File

@@ -315,7 +315,6 @@ export default {
mounted () {
if (!this.$utils.tour.isFinishedTour('setting')) {
this.$utils.tour.startTour(this, 'setting')
this.$st.sendEv('教程', '开始', 'setting')
}
}
}

2
frontend/src/views/spider/SpiderDetail.vue
View File

@@ -209,7 +209,6 @@ export default {
if (!this.$utils.tour.isFinishedTour('spider-detail-config')) {
setTimeout(() => {
this.$utils.tour.startTour(this, 'spider-detail-config')
this.$st.sendEv('教程', '开始', 'spider-detail-config')
}, 100)
}
} else if (this.activeTabName === 'scrapy-settings') {
@@ -261,7 +260,6 @@ export default {
mounted () {
if (!this.$utils.tour.isFinishedTour('spider-detail')) {
this.$utils.tour.startTour(this, 'spider-detail')
this.$st.sendEv('教程', '开始', 'spider-detail')
}
}
}

2
frontend/src/views/spider/SpiderList.vue
View File

@@ -888,7 +888,6 @@ export default {
setTimeout(() => {
if (!this.$utils.tour.isFinishedTour('spider-list-add')) {
this.$utils.tour.startTour(this, 'spider-list-add')
this.$st.sendEv('教程', '开始', 'spider-list-add')
}
}, 300)
},
@@ -1242,7 +1241,6 @@ export default {
if (!this.$utils.tour.isFinishedTour('spider-list')) {
this.$utils.tour.startTour(this, 'spider-list')
this.$st.sendEv('教程', '开始', 'spider-list')
}
},
destroyed () {

1
frontend/src/views/task/TaskDetail.vue
View File

@@ -203,7 +203,6 @@ export default {
mounted () {
if (!this.$utils.tour.isFinishedTour('task-detail')) {
this.$utils.tour.startTour(this, 'task-detail')
this.$st.sendEv('教程', '开始', 'task-detail')
}
},
destroyed () {

1
frontend/src/views/task/TaskList.vue
View File

@@ -413,7 +413,6 @@ export default {
if (!this.$utils.tour.isFinishedTour('task-list')) {
this.$utils.tour.startTour(this, 'task-list')
this.$st.sendEv('教程', '开始', 'task-list')
}
},
destroyed () {

5
k8s/crawlab-master.yaml
View File

@@ -30,6 +30,7 @@ spec:
spec:
containers:
- image: tikazyq/crawlab:latest
imagePullPolicy: Always
name: crawlab
env:
- name: CRAWLAB_SERVER_MASTER
@@ -38,6 +39,10 @@ spec:
value: "mongo"
- name: CRAWLAB_REDIS_ADDRESS
value: "redis"
# - name: CRAWLAB_SERVER_LANG_NODE
# value: "Y"
# - name: CRAWLAB_SERVER_LANG_JAVA
# value: "Y"
ports:
- containerPort: 8080
name: crawlab

5
k8s/crawlab-worker.yaml
View File

@@ -17,6 +17,7 @@ spec:
spec:
containers:
- image: tikazyq/crawlab:latest
imagePullPolicy: Always
name: crawlab
env:
- name: CRAWLAB_SERVER_MASTER
@@ -25,3 +26,7 @@ spec:
value: "mongo"
- name: CRAWLAB_REDIS_ADDRESS
value: "redis"
# - name: CRAWLAB_SERVER_LANG_NODE
# value: "Y"
# - name: CRAWLAB_SERVER_LANG_JAVA
# value: "Y"

16
frontend/conf/crawlab.conf → nginx/crawlab.conf
View File

@@ -2,17 +2,17 @@ server {
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
#gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
gzip_vary off;
gzip_disable "MSIE [1-6]\.";
listen 8080;
root /app/dist;
index index.html;
client_max_body_size 50m;
listen 8080;
root /app/dist;
index index.html;
location /api/ {
rewrite /api/(.*) /$1 break;
proxy_pass http://localhost:8000/;
}
location /api/ {
rewrite /api/(.*) /$1 break;
proxy_pass http://localhost:8000/;
}
}