Merge pull request #168 from crawlab-team/develop

Develop
This commit is contained in:
Marvin Zhang
2019-08-24 17:30:14 +08:00
committed by GitHub
18 changed files with 496 additions and 38 deletions

9
Jenkinsfile vendored
View File

@@ -48,8 +48,11 @@ pipeline {
sh """
# 重启docker compose
cd ./jenkins/${ENV:GIT_BRANCH}
docker-compose stop | true
docker-compose up -d
docker-compose stop master | true
docker-compose rm -f master | true
docker-compose stop worker | true
docker-compose rm -f worker | true
docker-compose up -d | true
"""
}
}
@@ -57,7 +60,7 @@ pipeline {
steps {
echo 'Cleanup...'
sh """
docker rmi `docker images | grep '<none>' | grep -v IMAGE | awk '{ print \$3 }' | xargs`
docker rmi -f `docker images | grep '<none>' | grep -v IMAGE | awk '{ print \$3 }' | xargs`
"""
}
}

View File

@@ -1,9 +1,11 @@
package database
import (
"errors"
"fmt"
"github.com/apex/log"
"github.com/gomodule/redigo/redis"
"time"
"unsafe"
)
@@ -23,22 +25,38 @@ func (c *Subscriber) Connect() {
c.client = redis.PubSubConn{Conn: conn}
c.cbMap = make(map[string]SubscribeCallback)
go func() {
//retry connect redis 5 times, or panic
index := 0
go func(i int) {
for {
log.Debug("wait...")
switch res := c.client.Receive().(type) {
case redis.Message:
i = 0
channel := (*string)(unsafe.Pointer(&res.Channel))
message := (*string)(unsafe.Pointer(&res.Data))
c.cbMap[*channel](*channel, *message)
case redis.Subscription:
fmt.Printf("%s: %s %d\n", res.Channel, res.Kind, res.Count)
case error:
log.Error("error handle...")
log.Error("error handle redis connection...")
time.Sleep(2 * time.Second)
if i > 5 {
panic(errors.New("redis connection failed too many times, panic"))
}
con, err := GetRedisConn()
if err != nil {
log.Error("redis dial failed")
continue
}
c.client = redis.PubSubConn{Conn: con}
i += 1
continue
}
}
}()
}(index)
}

View File

@@ -42,7 +42,7 @@ var NodeList = []model.Node{
var TaskList = []model.Task{
{
Id: "1234",
SpiderId: bson.ObjectId("xx429e6c19f7abede924fee2"),
SpiderId: bson.ObjectId("5d429e6c19f7abede924fee2"),
StartTs: time.Now(),
FinishTs: time.Now(),
Status: "进行中",
@@ -61,7 +61,7 @@ var TaskList = []model.Task{
},
{
Id: "5678",
SpiderId: bson.ObjectId("xx429e6c19f7abede924fddf"),
SpiderId: bson.ObjectId("5d429e6c19f7abede924fee2"),
StartTs: time.Now(),
FinishTs: time.Now(),
Status: "进行中",

View File

@@ -35,6 +35,12 @@ func init() {
app.PUT("/schedules", PutSchedule) // 创建定时任务
app.POST("/schedules/:id", PostSchedule) // 修改定时任务
app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务
app.GET("/tasks", GetTaskList) // 任务列表
app.GET("/tasks/:id", GetTask) // 任务详情
app.PUT("/tasks", PutTask) // 派发任务
app.DELETE("/tasks/:id", DeleteTask) // 删除任务
app.GET("/tasks/:id/results",GetTaskResults) // 任务结果
app.GET("/tasks/:id/results/download", DownloadTaskResultsCsv) // 下载任务结果
}
//mock test, test data in ./mock

View File

@@ -10,7 +10,6 @@ import (
"strings"
"testing"
"time"
"ucloudBilling/ucloud/log"
)
func TestGetScheduleList(t *testing.T) {
@@ -58,7 +57,6 @@ func TestDeleteSchedule(t *testing.T) {
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
log.Info(w.Body.String())
if err != nil {
t.Fatal("Unmarshal resp failed")
}
@@ -89,7 +87,6 @@ func TestPostSchedule(t *testing.T) {
var resp Response
var mongoId = "5d429e6c19f7abede924fee2"
body,_ := json.Marshal(newItem)
log.Info(strings.NewReader(string(body)))
w := httptest.NewRecorder()
req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body)))
app.ServeHTTP(w, req)
@@ -125,7 +122,6 @@ func TestPutSchedule(t *testing.T) {
var resp Response
body,_ := json.Marshal(newItem)
log.Info(strings.NewReader(string(body)))
w := httptest.NewRecorder()
req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body)))
app.ServeHTTP(w, req)

View File

@@ -1 +1,224 @@
package mock
package mock
import (
"bytes"
"crawlab/constants"
"crawlab/model"
"crawlab/utils"
"encoding/csv"
"fmt"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"github.com/satori/go.uuid"
"net/http"
)
type TaskListRequestData struct {
PageNum int `form:"page_num"`
PageSize int `form:"page_size"`
NodeId string `form:"node_id"`
SpiderId string `form:"spider_id"`
}
type TaskResultsRequestData struct {
PageNum int `form:"page_num"`
PageSize int `form:"page_size"`
}
func GetTaskList(c *gin.Context) {
// 绑定数据
data := TaskListRequestData{}
if err := c.ShouldBindQuery(&data); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
if data.PageNum == 0 {
data.PageNum = 1
}
if data.PageSize == 0 {
data.PageNum = 10
}
// 过滤条件
query := bson.M{}
if data.NodeId != "" {
query["node_id"] = bson.ObjectIdHex(data.NodeId)
}
if data.SpiderId != "" {
query["spider_id"] = bson.ObjectIdHex(data.SpiderId)
}
// 获取任务列表
tasks := TaskList
// 获取总任务数
total := len(TaskList)
c.JSON(http.StatusOK, ListResponse{
Status: "ok",
Message: "success",
Total: total,
Data: tasks,
})
}
func GetTask(c *gin.Context) {
id := c.Param("id")
var result model.Task
for _, task := range TaskList {
if task.Id == id {
result = task
}
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: result,
})
}
func PutTask(c *gin.Context) {
// 生成任务ID,generate task ID
id := uuid.NewV4()
// 绑定数据
var t model.Task
if err := c.ShouldBindJSON(&t); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
t.Id = id.String()
t.Status = constants.StatusPending
// 如果没有传入node_id则置为null
if t.NodeId.Hex() == "" {
t.NodeId = bson.ObjectIdHex(constants.ObjectIdNull)
}
// 将任务存入数据库,put the task into database
fmt.Println("put the task into database")
// 加入任务队列, put the task into task queue
fmt.Println("put the task into task queue")
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func DeleteTask(c *gin.Context) {
id := c.Param("id")
for _, task := range TaskList {
if task.Id == id {
fmt.Println("delete the task")
}
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func GetTaskResults(c *gin.Context) {
id := c.Param("id")
// 绑定数据
data := TaskResultsRequestData{}
if err := c.ShouldBindQuery(&data); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
// 获取任务
var task model.Task
for _, ta := range TaskList {
if ta.Id == id {
task = ta
}
}
fmt.Println(task)
// 获取结果
var results interface{}
total := len(TaskList)
c.JSON(http.StatusOK, ListResponse{
Status: "ok",
Message: "success",
Data: results,
Total: total,
})
}
func DownloadTaskResultsCsv(c *gin.Context) {
id := c.Param("id")
// 获取任务
var task model.Task
for _, ta := range TaskList {
if ta.Id == id {
task = ta
}
}
fmt.Println(task)
// 获取结果
var results []interface {
}
// 字段列表
var columns []string
if len(results) == 0 {
columns = []string{}
} else {
item := results[0].(bson.M)
for key := range item {
columns = append(columns, key)
}
}
// 缓冲
bytesBuffer := &bytes.Buffer{}
// 写入UTF-8 BOM避免使用Microsoft Excel打开乱码
bytesBuffer.Write([]byte("\xEF\xBB\xBF"))
writer := csv.NewWriter(bytesBuffer)
// 写入表头
if err := writer.Write(columns); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
// 写入内容
for _, result := range results {
// 将result转换为[]string
item := result.(bson.M)
var values []string
for _, col := range columns {
value := utils.InterfaceToString(item[col])
values = append(values, value)
}
// 写入数据
if err := writer.Write(values); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
// 此时才会将缓冲区数据写入
writer.Flush()
// 设置下载的文件名
c.Writer.Header().Set("Content-Disposition", "attachment;filename=data.csv")
// 设置文件类型以及输出数据
c.Data(http.StatusOK, "text/csv", bytesBuffer.Bytes())
}

138
backend/mock/task_test.go Normal file
View File

@@ -0,0 +1,138 @@
package mock
import (
"crawlab/model"
"encoding/json"
"github.com/globalsign/mgo/bson"
. "github.com/smartystreets/goconvey/convey"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestGetTaskList(t *testing.T) {
//var teskListRequestFrom = TaskListRequestData{
// PageNum: 2,
// PageSize: 10,
// NodeId: "434221grfsf",
// SpiderId: "fdfewqrftea",
//}
var resp ListResponse
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/tasks?PageNum=2&PageSize=10&NodeId=342dfsff&SpiderId=f8dsf", nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API GetNodeList", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
So(resp.Total, ShouldEqual, 2)
})
})
}
func TestGetTask(t *testing.T) {
var resp Response
var taskId = "1234"
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/tasks/"+taskId, nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API GetTask", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestPutTask(t *testing.T) {
var newItem = model.Task{
Id: "1234",
SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
StartTs: time.Now(),
FinishTs: time.Now(),
Status: "online",
NodeId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
LogPath: "./log",
Cmd: "scrapy crawl test",
Error: "",
ResultCount: 0,
WaitDuration: 10.0,
RuntimeDuration: 10,
TotalDuration: 20,
SpiderName: "test",
NodeName: "test",
CreateTs: time.Now(),
UpdateTs: time.Now(),
}
var resp Response
body, _ := json.Marshal(&newItem)
w := httptest.NewRecorder()
req, _ := http.NewRequest("PUT", "/tasks", strings.NewReader(string(body)))
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("unmarshal resp failed")
}
Convey("Test API PutTask", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestDeleteTask(t *testing.T) {
taskId := "1234"
var resp Response
w := httptest.NewRecorder()
req, _ := http.NewRequest("DELETE", "/tasks/"+taskId, nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("unmarshal resp failed")
}
Convey("Test API DeleteTask", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestGetTaskResults(t *testing.T) {
//var teskListResultFrom = TaskResultsRequestData{
// PageNum: 2,
// PageSize: 1,
//}
taskId := "1234"
var resp ListResponse
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/tasks/"+taskId+"/results?PageNum=2&PageSize=1", nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API GetNodeList", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
So(resp.Total, ShouldEqual, 2)
})
})
}

View File

@@ -0,0 +1,14 @@
<svg width="300" height="300" xmlns="http://www.w3.org/2000/svg">
<circle cx="150" cy="150" r="150" fill="#409eff">
</circle>
<circle cx="150" cy="150" r="110" fill="#fff">
</circle>
<circle cx="150" cy="150" r="70" fill="#409eff">
</circle>
<path d="
M 150,150
L 280,225
A 150,150 90 0 0 280,75
" fill="#409eff">
</path>
</svg>

After

Width:  |  Height:  |  Size: 393 B

View File

@@ -2,8 +2,8 @@
<div class="environment-list">
<el-row>
<div class="button-group">
<el-button type="primary" @click="addEnv" icon="el-icon-plus">{{$t('Add Environment Variables')}}</el-button>
<el-button type="success" @click="save">{{$t('Save')}}</el-button>
<el-button size="small" type="primary" @click="addEnv" icon="el-icon-plus">{{$t('Add Environment Variables')}}</el-button>
<el-button size="small" type="success" @click="save">{{$t('Save')}}</el-button>
</div>
</el-row>
<el-row>

View File

@@ -47,8 +47,8 @@
</el-form>
</el-row>
<el-row class="button-container" v-if="!isView">
<el-button v-if="isShowRun" type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
<el-button type="success" @click="onSave">{{$t('Save')}}</el-button>
<el-button size="small" v-if="isShowRun" type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
<el-button size="small" type="success" @click="onSave">{{$t('Save')}}</el-button>
</el-row>
</div>
</template>

View File

@@ -214,6 +214,7 @@ export default {
// 下拉框
User: '用户',
Logout: '退出登录',
Documentation: '文档',
// 选择
'Yes': '是',

View File

@@ -30,7 +30,7 @@
<el-dropdown class="documentation right">
<a href="https://tikazyq.github.io/crawlab-docs" target="_blank">
<font-awesome-icon :icon="['far', 'question-circle']"/>
<span style="margin-left: 5px;">文档</span>
<span style="margin-left: 5px;">{{$t('Documentation')}}</span>
</a>
</el-dropdown>
</div>

View File

@@ -4,7 +4,7 @@
<el-form ref="loginForm" :model="loginForm" :rules="loginRules" class="login-form" auto-complete="on"
label-position="left">
<h3 class="title">
CRAWLAB
<span><img style="width:48px;margin-bottom:-5px;margin-right:2px" src="../../assets/logo.svg"></span>RAWLAB
</h3>
<el-form-item prop="username" style="margin-bottom: 28px;">
<el-input
@@ -61,11 +61,18 @@
<img src="https://img.shields.io/badge/github-crawlab-blue">
</a>
</div>
<div class="lang">
<span @click="setLang('zh')" :class="lang==='zh'?'active':''">中文</span>
<span @click="setLang('en')" :class="lang==='en'?'active':''">English</span>
</div>
</el-form>
</div>
</template>
<script>
import {
mapState
} from 'vuex'
import { isValidUsername } from '../../utils/validate'
export default {
@@ -109,6 +116,9 @@ export default {
}
},
computed: {
...mapState('lang', [
'lang'
]),
isSignUp () {
return this.$route.path === '/signup'
},
@@ -149,6 +159,11 @@ export default {
onKeyEnter () {
const func = this.isSignUp ? this.handleSignup : this.handleLogin
func()
},
setLang (lang) {
window.localStorage.setItem('lang', lang)
this.$set(this.$i18n, 'locale', lang)
this.$store.commit('lang/SET_LANG', lang)
}
},
mounted () {
@@ -374,6 +389,7 @@ const initCanvas = () => {
color: #409EFF;
margin: 0px auto 20px auto;
text-align: center;
cursor: default;
}
.show-pwd {
@@ -407,5 +423,25 @@ const initCanvas = () => {
font-weight: 600;
}
}
.lang {
margin-top: 20px;
text-align: center;
span {
cursor: pointer;
margin: 10px;
color: #666;
font-size: 14px;
}
span.active {
font-weight: 600;
}
span:hover {
text-decoration: underline;
}
}
}
</style>

View File

@@ -165,7 +165,7 @@ export default {
{ name: 'ip', label: 'IP', width: '160' },
{ name: 'type', label: 'Type', width: '120' },
// { name: 'port', label: 'Port', width: '80' },
{ name: 'status', label: 'Status', width: '120', sortable: true },
{ name: 'status', label: 'Status', width: '120' },
{ name: 'description', label: 'Description', width: 'auto' }
],
nodeFormRules: {

View File

@@ -52,7 +52,7 @@
v-model="scheduleForm.cron"
:placeholder="$t('Cron')">
</el-input>
<el-button style="width:100px" type="primary" @click="onShowCronDialog">{{$t('生成Cron')}}</el-button>
<el-button size="small" style="width:100px" type="primary" @click="onShowCronDialog">{{$t('生成Cron')}}</el-button>
</el-form-item>
<el-form-item :label="$t('Execute Command')" prop="params">
<el-input v-model="spider.cmd"
@@ -70,8 +70,8 @@
</el-form-item>
</el-form>
<span slot="footer" class="dialog-footer">
<el-button @click="onCancel">{{$t('Cancel')}}</el-button>
<el-button type="primary" @click="onAddSubmit">{{$t('Submit')}}</el-button>
<el-button size="small" @click="onCancel">{{$t('Cancel')}}</el-button>
<el-button size="small" type="primary" @click="onAddSubmit">{{$t('Submit')}}</el-button>
</span>
</el-dialog>
@@ -84,7 +84,7 @@
<!--filter-->
<div class="filter">
<div class="right">
<el-button type="primary"
<el-button size="small" type="primary"
icon="el-icon-plus"
class="refresh"
@click="onAdd">

View File

@@ -89,10 +89,11 @@
:on-change="onUploadChange"
:on-success="onUploadSuccess"
:file-list="fileList">
<el-button type="primary" icon="el-icon-upload">{{$t('Upload')}}</el-button>
<el-button size="small" type="primary" icon="el-icon-upload">{{$t('Upload')}}</el-button>
</el-upload>
</el-form-item>
</el-form>
<el-alert type="error" title="爬虫文件请从根目录下开始压缩。" :closable="false"></el-alert>
</el-dialog>
<!--./customized spider dialog-->
@@ -114,7 +115,7 @@
<!--@change="onSearch">-->
<!--</el-input>-->
<div class="left">
<el-autocomplete v-model="filterSite"
<el-autocomplete size="small" v-model="filterSite"
:placeholder="$t('Site')"
clearable
:fetch-suggestions="fetchSiteSuggestions"
@@ -122,16 +123,16 @@
</el-autocomplete>
</div>
<div class="right">
<el-button v-if="false" type="primary" icon="fa fa-download" @click="openImportDialog">
<el-button size="small" v-if="false" type="primary" icon="fa fa-download" @click="openImportDialog">
{{$t('Import Spiders')}}
</el-button>
<el-button type="success"
<el-button size="small" type="success"
icon="el-icon-plus"
class="btn add"
@click="onAdd">
{{$t('Add Spider')}}
</el-button>
<el-button type="success"
<el-button size="small" type="success"
icon="el-icon-refresh"
class="btn refresh"
@click="onRefresh">
@@ -538,7 +539,7 @@ export default {
}
.table {
margin-top: 20px;
margin-top: 8px;
border-radius: 5px;
.el-button {

View File

@@ -4,7 +4,7 @@
<!--filter-->
<div class="filter">
<div class="left">
<el-select class="filter-select"
<el-select size="small" class="filter-select"
v-model="filter.node_id"
:placeholder="$t('Node')"
filterable
@@ -12,7 +12,7 @@
@change="onSelectNode">
<el-option v-for="op in nodeList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
<el-select class="filter-select"
<el-select size="small" class="filter-select"
v-model="filter.spider_id"
:placeholder="$t('Spider')"
filterable
@@ -20,7 +20,7 @@
@change="onSelectSpider">
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
<el-button type="success"
<el-button size="small" type="success"
icon="el-icon-search"
class="refresh"
@click="onRefresh">
@@ -119,7 +119,7 @@
:width="col.width">
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="150" fixed="right">
<el-table-column :label="$t('Action')" align="left" width="120" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('View')" placement="top">
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
@@ -172,11 +172,11 @@ export default {
{ name: 'node_name', label: 'Node', width: '120' },
{ name: 'spider_name', label: 'Spider', width: '120' },
{ name: 'status', label: 'Status', width: '120' },
{ name: 'create_ts', label: 'Create Time', width: '100' },
// { name: 'create_ts', label: 'Create Time', width: '100' },
{ name: 'start_ts', label: 'Start Time', width: '100' },
{ name: 'finish_ts', label: 'Finish Time', width: '100' },
{ name: 'wait_duration', label: 'Wait Duration (sec)', width: '80', align: 'right' },
{ name: 'runtime_duration', label: 'Runtime Duration (sec)', width: '80', align: 'right' },
{ name: 'wait_duration', label: 'Wait Duration (sec)', align: 'right' },
{ name: 'runtime_duration', label: 'Runtime Duration (sec)', align: 'right' },
{ name: 'total_duration', label: 'Total Duration (sec)', width: '80', align: 'right' },
{ name: 'result_count', label: 'Results Count', width: '80' }
// { name: 'avg_num_results', label: 'Average Results Count per Second', width: '80' }
@@ -348,7 +348,7 @@ export default {
}
.table {
margin-top: 20px;
margin-top: 8px;
border-radius: 5px;
.el-button {

View File

@@ -24,6 +24,12 @@
<!--./dialog-->
<el-card>
<div class="filter">
<div class="left"></div>
<div class="right">
<!--<el-button type="primary" size="small">新增用户</el-button>-->
</div>
</div>
<!--table-->
<el-table
:data="userList"
@@ -67,6 +73,7 @@
</template>
</el-table-column>
</el-table>
<div class="pagination">
<el-pagination
@current-change="onPageChange"
@@ -181,6 +188,21 @@ export default {
</script>
<style scoped>
.filter {
display: flex;
justify-content: space-between;
margin-bottom: 8px;
.filter-search {
width: 240px;
}
.right {
.btn {
margin-left: 10px;
}
}
}
.el-table {
border-radius: 5px;
}