Merge branch 'develop' of https://github.com/crawlab-team/crawlab into develop

backend/constants/context.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	ContextUser = "currentUser"
+)

backend/constants/errors.go

@@ -0,0 +1,8 @@
+package constants
+
+import "crawlab/errors"
+
+var (
+	//users
+	ErrorUserNotFound = errors.NewBusinessError(10001, "user not found.")
+)

backend/errors/errors.go

@@ -0,0 +1,43 @@
+package errors
+
+import "fmt"
+
+type Scope int
+
+const (
+	ScopeSystem   Scope = 1
+	ScopeBusiness Scope = 2
+)
+
+type OPError struct {
+	Message string
+	Code    int
+	Scope   Scope
+}
+
+func (O OPError) Error() string {
+	var scope string
+	switch O.Scope {
+	case ScopeSystem:
+		scope = "system"
+		break
+	case ScopeBusiness:
+		scope = "business"
+	}
+	return fmt.Sprintf("%s : %d -> %s.", scope, O.Code, O.Message)
+}
+
+func NewSystemOPError(code int, message string) *OPError {
+	return &OPError{
+		Message: message,
+		Code:    code,
+		Scope:   ScopeSystem,
+	}
+}
+func NewBusinessError(code int, message string) *OPError {
+	return &OPError{
+		Message: message,
+		Code:    code,
+		Scope:   ScopeBusiness,
+	}
+}

backend/go.mod

@@ -8,9 +8,13 @@ require (
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/gin-gonic/gin v1.4.0
 	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
+	github.com/go-playground/locales v0.12.1 // indirect
+	github.com/go-playground/universal-translator v0.16.0 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/leodido/go-urn v1.1.0 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/satori/go.uuid v1.2.0
 	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337
 	github.com/spf13/viper v1.4.0
+	gopkg.in/go-playground/validator.v9 v9.29.1
 )

backend/go.sum

@@ -39,6 +39,10 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
+github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
+github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
+github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -77,6 +81,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
+github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
@@ -202,6 +208,8 @@ gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXa
 gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
+gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc=
+gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=

backend/lib/validate_bridge/validator.go

@@ -0,0 +1,54 @@
+package validate_bridge
+
+import (
+	"reflect"
+	"sync"
+
+	"github.com/gin-gonic/gin/binding"
+	"gopkg.in/go-playground/validator.v9"
+)
+
+type DefaultValidator struct {
+	once     sync.Once
+	validate *validator.Validate
+}
+
+var _ binding.StructValidator = &DefaultValidator{validate: validator.New()}
+
+func (v *DefaultValidator) ValidateStruct(obj interface{}) error {
+	if kindOfData(obj) == reflect.Struct {
+
+		v.lazyinit()
+
+		if err := v.validate.Struct(obj); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (v *DefaultValidator) Engine() interface{} {
+	v.lazyinit()
+	return v.validate
+}
+
+func (v *DefaultValidator) lazyinit() {
+	v.once.Do(func() {
+		v.validate = validator.New()
+		v.validate.SetTagName("binding")
+
+		// add any custom validations etc. here
+	})
+}
+
+func kindOfData(data interface{}) reflect.Kind {
+
+	value := reflect.ValueOf(data)
+	valueType := value.Kind()
+
+	if valueType == reflect.Ptr {
+		valueType = value.Elem().Kind()
+	}
+	return valueType
+}

backend/main.go

@@ -3,16 +3,19 @@ package main
 import (
 	"crawlab/config"
 	"crawlab/database"
+	"crawlab/lib/validate_bridge"
 	"crawlab/middlewares"
 	"crawlab/routes"
 	"crawlab/services"
 	"github.com/apex/log"
 	"github.com/gin-gonic/gin"
+	"github.com/gin-gonic/gin/binding"
 	"github.com/spf13/viper"
 	"runtime/debug"
 )
 
 func main() {
+	binding.Validator = new(validate_bridge.DefaultValidator)
 	app := gin.Default()
 
 	// 初始化配置

backend/middlewares/auth.go

@@ -46,6 +46,7 @@ func AuthorizationMiddleware() gin.HandlerFunc {
 				return
 			}
 		}
+		c.Set(constants.ContextUser, &user)
 
 		// 校验成功
 		c.Next()

backend/mock/schedule.go

@@ -113,7 +113,7 @@ func PutSchedule(c *gin.Context) {
 func DeleteSchedule(c *gin.Context) {
 	id := bson.ObjectIdHex("5d429e6c19f7abede924fee2")
 	for _, sch := range scheduleList {
-		if sch.Id == bson.ObjectId(id) {
+		if sch.Id == id {
 			fmt.Println("delete a schedule")
 		}
 	}

backend/routes/user.go

@@ -4,6 +4,7 @@ import (
 	"crawlab/constants"
 	"crawlab/model"
 	"crawlab/services"
+	"crawlab/services/context"
 	"crawlab/utils"
 	"github.com/gin-gonic/gin"
 	"github.com/globalsign/mgo/bson"
@@ -171,7 +172,7 @@ func Login(c *gin.Context) {
 	}
 
 	// 获取token
-	tokenStr, err := services.GetToken(user.Username)
+	tokenStr, err := services.MakeToken(&user)
 	if err != nil {
 		HandleError(http.StatusUnauthorized, c, errors.New("not authorized"))
 		return
@@ -185,20 +186,16 @@ func Login(c *gin.Context) {
 }
 
 func GetMe(c *gin.Context) {
-	// 获取token string
-	tokenStr := c.GetHeader("Authorization")
-
-	// 校验token
-	user, err := services.CheckToken(tokenStr)
-	if err != nil {
-		HandleError(http.StatusUnauthorized, c, errors.New("not authorized"))
+	ctx := context.WithGinContext(c)
+	user := ctx.User()
+	if user == nil {
+		ctx.FailedWithError(constants.ErrorUserNotFound, http.StatusUnauthorized)
 		return
 	}
-	user.Password = ""
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    user,
-	})
+	ctx.Success(struct {
+		*model.User
+		Password string `json:"password,omitempty"`
+	}{
+		User: user,
+	}, nil)
 }

backend/services/context/context.go

@@ -0,0 +1,73 @@
+package context
+
+import (
+	"crawlab/constants"
+	"crawlab/errors"
+	"crawlab/model"
+	"fmt"
+	"github.com/apex/log"
+	"github.com/gin-gonic/gin"
+	errors2 "github.com/pkg/errors"
+	"net/http"
+	"runtime/debug"
+)
+
+type Context struct {
+	*gin.Context
+}
+
+func (c *Context) User() *model.User {
+	userIfe, exists := c.Get(constants.ContextUser)
+	if !exists {
+		return nil
+	}
+	user, ok := userIfe.(*model.User)
+	if !ok {
+		return nil
+	}
+	return user
+}
+func (c *Context) Success(data interface{}, meta interface{}) {
+	if meta == nil {
+		meta = gin.H{}
+	}
+	if data == nil {
+		data = gin.H{}
+	}
+	c.JSON(http.StatusOK, gin.H{
+		"status":  "ok",
+		"message": "success",
+		"data":    data,
+		"error":   "",
+	})
+}
+func (c *Context) FailedWithError(err error, httpCode ...int) {
+
+	var code = 200
+	if len(httpCode) > 0 {
+		code = httpCode[0]
+	}
+	log.Errorf("handle error:" + err.Error())
+	debug.PrintStack()
+	switch errors2.Cause(err).(type) {
+	case errors.OPError:
+		c.AbortWithStatusJSON(code, gin.H{
+			"status":  "ok",
+			"message": "error",
+			"error":   err.Error(),
+		})
+		break
+	default:
+		fmt.Println("deprecated....")
+		c.AbortWithStatusJSON(code, gin.H{
+			"status":  "ok",
+			"message": "error",
+			"error":   err.Error(),
+		})
+	}
+
+}
+
+func WithGinContext(context *gin.Context) *Context {
+	return &Context{Context: context}
+}

backend/services/log.go

@@ -34,13 +34,17 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) {
 		return nil, err
 	}
 	defer f.Close()
-	logBuf := make([]byte, 2048)
+
+	const bufLen = 2048
+	logBuf := make([]byte, bufLen)
+
 	off := int64(0)
 	if fi.Size() > int64(len(logBuf)) {
 		off = fi.Size() - int64(len(logBuf))
 	}
 	n, err := f.ReadAt(logBuf, off)
-	// 到文件结尾会有EOF的报错
+
+	//到文件结尾会有EOF标识
 	if err != nil && err.Error() != "EOF" {
 		log.Error(err.Error())
 		debug.PrintStack()

backend/services/log_test.go

@@ -2,9 +2,11 @@ package services
 
 import (
 	"crawlab/config"
+	"fmt"
 	"github.com/apex/log"
 	. "github.com/smartystreets/goconvey/convey"
 	"github.com/spf13/viper"
+	"os"
 	"testing"
 )
 
@@ -20,3 +22,29 @@ func TestDeleteLogPeriodically(t *testing.T) {
 		DeleteLogPeriodically()
 	})
 }
+
+func TestGetLocalLog(t *testing.T) {
+	//create a log file for test
+	logPath := "../logs/crawlab/test.log"
+	f, err := os.Create(logPath)
+	defer f.Close()
+	if err != nil {
+		fmt.Println(err.Error())
+
+	} else {
+		_, err = f.Write([]byte("This is for test"))
+	}
+
+	Convey("Test GetLocalLog", t, func() {
+		Convey("Test response", func() {
+			logStr, err := GetLocalLog(logPath)
+			log.Info(string(logStr))
+			fmt.Println(err)
+			So(err, ShouldEqual, nil)
+
+		})
+	})
+	//delete the test log file
+	os.Remove(logPath)
+
+}

backend/services/spider.go

@@ -230,7 +230,7 @@ func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCre
 	for {
 		switch nr, err := f.Read(s[:]); true {
 		case nr < 0:
-			fmt.Fprintf(os.Stderr, "cat: error reading: %s\n", err.Error())
+			_, _ = fmt.Fprintf(os.Stderr, "cat: error reading: %s\n", err.Error())
 			debug.PrintStack()
 		case nr == 0: // EOF
 			return nil
@@ -238,7 +238,6 @@ func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCre
 			handle(s[0:nr], fileCreate)
 		}
 	}
-	return nil
 }
 
 // 发布所有爬虫

backend/services/user.go

@@ -5,11 +5,9 @@ import (
 	"crawlab/model"
 	"crawlab/utils"
 	"errors"
-	"github.com/apex/log"
 	"github.com/dgrijalva/jwt-go"
 	"github.com/globalsign/mgo/bson"
 	"github.com/spf13/viper"
-	"runtime/debug"
 	"time"
 )
 
@@ -24,28 +22,38 @@ func InitUserService() error {
 	}
 	return nil
 }
-
-func GetToken(username string) (tokenStr string, err error) {
-	user, err := model.GetUserByUsername(username)
-	if err != nil {
-		log.Errorf(err.Error())
-		debug.PrintStack()
-		return
-	}
-
+func MakeToken(user *model.User) (tokenStr string, err error) {
 	token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
 		"id":       user.Id,
 		"username": user.Username,
 		"nbf":      time.Now().Unix(),
 	})
 
-	tokenStr, err = token.SignedString([]byte(viper.GetString("server.secret")))
-	if err != nil {
-		return
-	}
-	return
+	return token.SignedString([]byte(viper.GetString("server.secret")))
+
 }
 
+//func GetToken(username string) (tokenStr string, err error) {
+//	user, err := model.GetUserByUsername(username)
+//	if err != nil {
+//		log.Errorf(err.Error())
+//		debug.PrintStack()
+//		return
+//	}
+//
+//	token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
+//		"id":       user.Id,
+//		"username": user.Username,
+//		"nbf":      time.Now().Unix(),
+//	})
+//
+//	tokenStr, err = token.SignedString([]byte(viper.GetString("server.secret")))
+//	if err != nil {
+//		return
+//	}
+//	return
+//}
+
 func SecretFunc() jwt.Keyfunc {
 	return func(token *jwt.Token) (interface{}, error) {
 		return []byte(viper.GetString("server.secret")), nil

frontend/src/api/request.js

@@ -3,13 +3,13 @@ import router from '../router'
 
 let baseUrl = process.env.VUE_APP_BASE_URL ? process.env.VUE_APP_BASE_URL : 'http://localhost:8000'
 
-const request = (method, path, params, data, others = {}) => {
-  return new Promise((resolve, reject) => {
+const request = async (method, path, params, data, others = {}) => {
+  try {
     const url = baseUrl + path
     const headers = {
       'Authorization': window.localStorage.getItem('token')
     }
-    axios({
+    const response = await axios({
       method,
       url,
       params,
@@ -17,15 +17,37 @@ const request = (method, path, params, data, others = {}) => {
       headers,
       ...others
     })
-      .then(resolve)
-      .catch(error => {
-        console.log(error)
-        if (error.response.status === 401) {
-          router.push('/login')
-        }
-        reject(error)
-      })
-  })
+    // console.log(response)
+    return response
+  } catch (e) {
+    if (e.response.status === 401 && router.currentRoute.path !== '/login') {
+      router.push('/login')
+    }
+    await Promise.reject(e)
+  }
+
+  // return new Promise((resolve, reject) => {
+  //   const url = baseUrl + path
+  //   const headers = {
+  //     'Authorization': window.localStorage.getItem('token')
+  //   }
+  //   axios({
+  //     method,
+  //     url,
+  //     params,
+  //     data,
+  //     headers,
+  //     ...others
+  //   })
+  //     .then(resolve)
+  //     .catch(error => {
+  //       console.log(error)
+  //       if (error.response.status === 401) {
+  //         router.push('/login')
+  //       }
+  //       reject(error)
+  //     })
+  // })
 }
 
 const get = (path, params) => {

frontend/src/i18n/zh.js

@@ -247,7 +247,7 @@ export default {
   'username already exists': '用户名已存在',
   'Deleted successfully': '成功删除',
   'Saved successfully': '成功保存',
-
+  'English': 'English',
   // 登录
   'Sign in': '登录',
   'Sign-in': '登录',
@@ -266,5 +266,20 @@ export default {
   'admin': '管理用户',
   'Role': '角色',
   'Edit User': '更改用户',
-  'Users': '用户'
+  'Users': '用户',
+  tagsView: {
+    closeOthers: '关闭其他',
+    close: '关闭',
+    refresh: '刷新',
+    closeAll: '关闭所有'
+  },
+  nodeList: {
+    type: '节点类型'
+  },
+  schedules: {
+    cron: 'Cron',
+    add_cron: '生成Cron',
+    // Cron Format: [second] [minute] [hour] [day of month] [month] [day of week]
+    cron_format: 'Cron 格式: [秒] [分] [小时] [日] [月] [周]'
+  }
 }

frontend/src/router/index.js

@@ -46,7 +46,6 @@ export const constantRouterMap = [
     ]
   },
   {
-    name: 'Node',
     path: '/nodes',
     component: Layout,
     meta: {
@@ -76,7 +75,6 @@ export const constantRouterMap = [
     ]
   },
   {
-    name: 'Spider',
     path: '/spiders',
     component: Layout,
     meta: {
@@ -106,7 +104,6 @@ export const constantRouterMap = [
     ]
   },
   {
-    name: 'Task',
     path: '/tasks',
     component: Layout,
     meta: {
@@ -136,7 +133,6 @@ export const constantRouterMap = [
     ]
   },
   {
-    name: 'Schedule',
     path: '/schedules',
     component: Layout,
     meta: {
@@ -157,7 +153,6 @@ export const constantRouterMap = [
     ]
   },
   {
-    name: 'Site',
     path: '/sites',
     component: Layout,
     hidden: true,
@@ -178,7 +173,6 @@ export const constantRouterMap = [
     ]
   },
   {
-    name: 'User',
     path: '/users',
     component: Layout,
     meta: {

frontend/src/views/node/NodeList.vue

@@ -163,7 +163,7 @@ export default {
       columns: [
         { name: 'name', label: 'Name', width: '220' },
         { name: 'ip', label: 'IP', width: '160' },
-        { name: 'type', label: 'Type', width: '120' },
+        { name: 'type', label: 'nodeList.type', width: '120' },
         // { name: 'port', label: 'Port', width: '80' },
         { name: 'status', label: 'Status', width: '120' },
         { name: 'description', label: 'Description', width: 'auto' }

frontend/src/views/schedule/ScheduleList.vue

@@ -38,21 +38,21 @@
             </el-option>
           </el-select>
         </el-form-item>
-        <el-form-item :label="$t('Cron')" prop="cron" :rules="cronRules" required>
+        <el-form-item :label="$t('schedules.cron')" prop="cron" :rules="cronRules" required>
           <template slot="label">
-            <el-tooltip :content="$t('Cron Format: [second] [minute] [hour] [day of month] [month] [day of week]')"
+            <el-tooltip :content="$t('schedules.cron_format')"
                         placement="top">
               <span>
-                {{$t('Cron')}}
+                {{$t('schedules.cron')}}
                 <i class="fa fa-exclamation-circle"></i>
               </span>
             </el-tooltip>
           </template>
           <el-input style="width:calc(100% - 100px);padding-right:10px"
                     v-model="scheduleForm.cron"
-                    :placeholder="$t('Cron')">
+                    :placeholder="$t('schedules.cron')">
           </el-input>
-          <el-button size="small" style="width:100px" type="primary" @click="onShowCronDialog">{{$t('生成Cron')}}</el-button>
+          <el-button size="small" style="width:100px" type="primary" @click="onShowCronDialog">{{$t('schedules.add_cron')}}</el-button>
         </el-form-item>
         <el-form-item :label="$t('Execute Command')" prop="params">
           <el-input v-model="spider.cmd"
@@ -156,7 +156,7 @@ export default {
     return {
       columns: [
         { name: 'name', label: 'Name', width: '180' },
-        { name: 'cron', label: 'Cron', width: '120' },
+        { name: 'cron', label: 'schedules.cron', width: '120' },
         { name: 'node_name', label: 'Node', width: '150' },
         { name: 'spider_name', label: 'Spider', width: '150' },
         { name: 'description', label: 'Description', width: 'auto' }