git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #420 from crawlab-team/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2020-01-03 16:23:11 +08:00
committed by GitHub
parent 5e88ad48ee 9374493fd2
commit fe8ae218d1
35 changed files with 3480 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@@ -1,3 +1,10 @@
# 0.4.3 (unknown)
### Features / Enhancement
- **Dependency Installation**. Allow users to install/uninstall dependencies and add programming languages (Node.js only for now) on the platform web interface.
- **Pre-install Programming Languages in Docker**. Allow Docker users to set `CRAWLAB_SERVER_LANG_NODE` as `Y` to pre-install `Node.js` environments.
# 0.4.2 (2019-12-26)
### Features / Enhancement
- **Disclaimer**. Added page for Disclaimer.

5
Dockerfile
View File

@@ -27,6 +27,9 @@ ADD . /app
# set as non-interactive
ENV DEBIAN_FRONTEND noninteractive
# set CRAWLAB_IS_DOCKER
ENV CRAWLAB_IS_DOCKER Y
# install packages
RUN apt-get update \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip \
@@ -56,4 +59,4 @@ EXPOSE 8080
EXPOSE 8000
# start backend
CMD ["/bin/sh", "/app/docker_init.sh"]
CMD ["/bin/bash", "/app/docker_init.sh"]

2
Dockerfile.local
View File

@@ -57,4 +57,4 @@ EXPOSE 8080
EXPOSE 8000
# start backend
CMD ["/bin/sh", "/app/docker_init.sh"]
CMD ["/bin/bash", "/app/docker_init.sh"]

5
backend/conf/config.yml
View File

@@ -26,12 +26,15 @@ server:
# mac地址 或者 ip地址，如果是ip，则需要手动指定IP
type: "mac"
ip: ""
lang: # 安装语言环境, Y 为安装，N 为不安装，只对 Docker 有效
python: "Y"
node: "N"
spider:
path: "/app/spiders"
task:
workers: 4
other:
tmppath: "/tmp"
version: 0.4.2
version: 0.4.3
setting:
allowRegister: "N"

9
backend/constants/rpc.go Normal file
View File

@@ -0,0 +1,9 @@
package constants
const (
RpcInstallLang = "install_lang"
RpcInstallDep = "install_dep"
RpcUninstallDep = "uninstall_dep"
RpcGetDepList = "get_dep_list"
RpcGetInstalledDepList = "get_installed_dep_list"
)

6
backend/constants/system.go
View File

@@ -5,3 +5,9 @@ const (
Linux = "linux"
Darwin = "darwin"
)
const (
Python = "python"
Nodejs = "node"
Java = "java"
)

40
backend/database/redis.go
View File

@@ -36,6 +36,19 @@ func (r *Redis) RPush(collection string, value interface{}) error {
defer utils.Close(c)
if _, err := c.Do("RPUSH", collection, value); err != nil {
log.Error(err.Error())
debug.PrintStack()
return err
}
return nil
}
func (r *Redis) LPush(collection string, value interface{}) error {
c := r.pool.Get()
defer utils.Close(c)
if _, err := c.Do("RPUSH", collection, value); err != nil {
log.Error(err.Error())
debug.PrintStack()
return err
}
@@ -58,6 +71,7 @@ func (r *Redis) HSet(collection string, key string, value string) error {
defer utils.Close(c)
if _, err := c.Do("HSET", collection, key, value); err != nil {
log.Error(err.Error())
debug.PrintStack()
return err
}
@@ -70,6 +84,8 @@ func (r *Redis) HGet(collection string, key string) (string, error) {
value, err2 := redis.String(c.Do("HGET", collection, key))
if err2 != nil {
log.Error(err2.Error())
debug.PrintStack()
return value, err2
}
return value, nil
@@ -80,6 +96,8 @@ func (r *Redis) HDel(collection string, key string) error {
defer utils.Close(c)
if _, err := c.Do("HDEL", collection, key); err != nil {
log.Error(err.Error())
debug.PrintStack()
return err
}
return nil
@@ -91,11 +109,29 @@ func (r *Redis) HKeys(collection string) ([]string, error) {
value, err2 := redis.Strings(c.Do("HKeys", collection))
if err2 != nil {
log.Error(err2.Error())
debug.PrintStack()
return []string{}, err2
}
return value, nil
}
func (r *Redis) BRPop(collection string, timeout int) (string, error) {
if timeout <= 0 {
timeout = 60
}
c := r.pool.Get()
defer utils.Close(c)
values, err := redis.Strings(c.Do("BRPOP", collection, timeout))
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return "", err
}
return values[1], nil
}
func NewRedisPool() *redis.Pool {
var address = viper.GetString("redis.address")
var port = viper.GetString("redis.port")
@@ -112,8 +148,8 @@ func NewRedisPool() *redis.Pool {
Dial: func() (conn redis.Conn, e error) {
return redis.DialURL(url,
redis.DialConnectTimeout(time.Second*10),
redis.DialReadTimeout(time.Second*10),
redis.DialWriteTimeout(time.Second*15),
redis.DialReadTimeout(time.Second*600),
redis.DialWriteTimeout(time.Second*10),
)
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {

15
backend/entity/system.go
View File

@@ -13,3 +13,18 @@ type Executable struct {
FileName string `json:"file_name"`
DisplayName string `json:"display_name"`
}
type Lang struct {
Name string `json:"name"`
ExecutableName string `json:"executable_name"`
ExecutablePath string `json:"executable_path"`
DepExecutablePath string `json:"dep_executable_path"`
Installed bool `json:"installed"`
}
type Dependency struct {
Name string `json:"name"`
Version string `json:"version"`
Description string `json:"description"`
Installed bool `json:"installed"`
}

1
backend/go.mod
View File

@@ -11,6 +11,7 @@ require (
github.com/go-playground/locales v0.12.1 // indirect
github.com/go-playground/universal-translator v0.16.0 // indirect
github.com/gomodule/redigo v2.0.0+incompatible
github.com/imroc/req v0.2.4
github.com/leodido/go-urn v1.1.0 // indirect
github.com/pkg/errors v0.8.1
github.com/satori/go.uuid v1.2.0

2
backend/go.sum
View File

@@ -66,6 +66,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/imroc/req v0.2.4 h1:8XbvaQpERLAJV6as/cB186DtH5f0m5zAOtHEaTQ4ac0=
github.com/imroc/req v0.2.4/go.mod h1:J9FsaNHDTIVyW/b5r6/Df5qKEEEq2WzZKIgKSajd1AE=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=

61
backend/main.go
View File

@@ -31,24 +31,24 @@ func main() {
log.Error("init config error:" + err.Error())
panic(err)
}
log.Info("初始化配置成功")
log.Info("initialized config successfully")
// 初始化日志设置
logLevel := viper.GetString("log.level")
if logLevel != "" {
log.SetLevelFromString(logLevel)
}
log.Info("初始化日志设置成功")
log.Info("initialized log config successfully")
if viper.GetString("log.isDeletePeriodically") == "Y" {
err := services.InitDeleteLogPeriodically()
if err != nil {
log.Error("Init DeletePeriodically Failed")
log.Error("init DeletePeriodically failed")
panic(err)
}
log.Info("初始化定期清理日志配置成功")
log.Info("initialized periodically cleaning log successfully")
} else {
log.Info("默认未开启定期清理日志配置")
log.Info("periodically cleaning log is switched off")
}
// 初始化Mongodb数据库
@@ -57,7 +57,7 @@ func main() {
debug.PrintStack()
panic(err)
}
log.Info("初始化Mongodb数据库成功")
log.Info("initialized MongoDB successfully")
// 初始化Redis数据库
if err := database.InitRedis(); err != nil {
@@ -65,7 +65,7 @@ func main() {
debug.PrintStack()
panic(err)
}
log.Info("初始化Redis数据库成功")
log.Info("initialized Redis successfully")
if model.IsMaster() {
// 初始化定时任务
@@ -74,8 +74,8 @@ func main() {
debug.PrintStack()
panic(err)
}
log.Info("初始化定时任务成功")
}
log.Info("initialized schedule successfully")
// 初始化任务执行器
if err := services.InitTaskExecutor(); err != nil {
@@ -83,14 +83,14 @@ func main() {
debug.PrintStack()
panic(err)
}
log.Info("初始化任务执行器成功")
log.Info("initialized task executor successfully")
// 初始化节点服务
if err := services.InitNodeService(); err != nil {
log.Error("init node service error:" + err.Error())
panic(err)
}
log.Info("初始化节点配置成功")
log.Info("initialized node service successfully")
// 初始化爬虫服务
if err := services.InitSpiderService(); err != nil {
@@ -98,7 +98,7 @@ func main() {
debug.PrintStack()
panic(err)
}
log.Info("初始化爬虫服务成功")
log.Info("initialized spider service successfully")
// 初始化用户服务
if err := services.InitUserService(); err != nil {
@@ -106,7 +106,23 @@ func main() {
debug.PrintStack()
panic(err)
}
log.Info("初始化用户服务成功")
log.Info("initialized user service successfully")
// 初始化依赖服务
if err := services.InitDepsFetcher(); err != nil {
log.Error("init dependency fetcher error:" + err.Error())
debug.PrintStack()
panic(err)
}
log.Info("initialized dependency fetcher successfully")
// 初始化RPC服务
if err := services.InitRpcService(); err != nil {
log.Error("init rpc service error:" + err.Error())
debug.PrintStack()
panic(err)
}
log.Info("initialized rpc service successfully")
// 以下为主节点服务
if model.IsMaster() {
@@ -122,12 +138,18 @@ func main() {
{
// 路由
// 节点
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
// 爬虫
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
@@ -184,6 +206,9 @@ func main() {
authGroup.GET("/me", routes.GetMe) // 获取自己账户
// release版本
authGroup.GET("/version", routes.GetVersion) // 获取发布的版本
// 系统
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
}
}

315
backend/routes/system.go Normal file
View File

@@ -0,0 +1,315 @@
package routes
import (
"crawlab/constants"
"crawlab/entity"
"crawlab/services"
"fmt"
"github.com/gin-gonic/gin"
"net/http"
"strings"
)
func GetLangList(c *gin.Context) {
nodeId := c.Param("id")
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: services.GetLangList(nodeId),
})
}
func GetDepList(c *gin.Context) {
nodeId := c.Param("id")
lang := c.Query("lang")
depName := c.Query("dep_name")
var depList []entity.Dependency
if lang == constants.Python {
list, err := services.GetPythonDepList(nodeId, depName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
depList = list
} else if lang == constants.Nodejs {
list, err := services.GetNodejsDepList(nodeId, depName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
depList = list
} else {
HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang))
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: depList,
})
}
func GetInstalledDepList(c *gin.Context) {
nodeId := c.Param("id")
lang := c.Query("lang")
var depList []entity.Dependency
if lang == constants.Python {
if services.IsMasterNode(nodeId) {
list, err := services.GetPythonLocalInstalledDepList(nodeId)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
depList = list
} else {
list, err := services.GetPythonRemoteInstalledDepList(nodeId)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
depList = list
}
} else if lang == constants.Nodejs {
if services.IsMasterNode(nodeId) {
list, err := services.GetNodejsLocalInstalledDepList(nodeId)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
depList = list
} else {
list, err := services.GetNodejsRemoteInstalledDepList(nodeId)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
depList = list
}
} else {
HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang))
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: depList,
})
}
func GetAllDepList(c *gin.Context) {
lang := c.Param("lang")
depName := c.Query("dep_name")
// 获取所有依赖列表
var list []string
if lang == constants.Python {
_list, err := services.GetPythonDepListFromRedis()
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
list = _list
} else {
HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang))
return
}
// 过滤依赖列表
var depList []string
for _, name := range list {
if strings.HasPrefix(strings.ToLower(name), strings.ToLower(depName)) {
depList = append(depList, name)
}
}
// 只取前20
var returnList []string
for i, name := range depList {
if i >= 10 {
break
}
returnList = append(returnList, name)
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: returnList,
})
}
func InstallDep(c *gin.Context) {
type ReqBody struct {
Lang string `json:"lang"`
DepName string `json:"dep_name"`
}
nodeId := c.Param("id")
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
if reqBody.Lang == constants.Python {
if services.IsMasterNode(nodeId) {
_, err := services.InstallPythonLocalDep(reqBody.DepName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
} else {
_, err := services.InstallPythonRemoteDep(nodeId, reqBody.DepName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
} else if reqBody.Lang == constants.Nodejs {
if services.IsMasterNode(nodeId) {
_, err := services.InstallNodejsLocalDep(reqBody.DepName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
} else {
_, err := services.InstallNodejsRemoteDep(nodeId, reqBody.DepName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
} else {
HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang))
return
}
// TODO: check if install is successful
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func UninstallDep(c *gin.Context) {
type ReqBody struct {
Lang string `json:"lang"`
DepName string `json:"dep_name"`
}
nodeId := c.Param("id")
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
}
if reqBody.Lang == constants.Python {
if services.IsMasterNode(nodeId) {
_, err := services.UninstallPythonLocalDep(reqBody.DepName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
} else {
_, err := services.UninstallPythonRemoteDep(nodeId, reqBody.DepName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
} else if reqBody.Lang == constants.Nodejs {
if services.IsMasterNode(nodeId) {
_, err := services.UninstallNodejsLocalDep(reqBody.DepName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
} else {
_, err := services.UninstallNodejsRemoteDep(nodeId, reqBody.DepName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
} else {
HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang))
return
}
// TODO: check if uninstall is successful
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func GetDepJson(c *gin.Context) {
depName := c.Param("dep_name")
lang := c.Param("lang")
var dep entity.Dependency
if lang == constants.Python {
_dep, err := services.FetchPythonDepInfo(depName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
}
dep = _dep
} else {
HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang))
return
}
c.Header("Cache-Control", "max-age=86400")
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: dep,
})
}
func InstallLang(c *gin.Context) {
type ReqBody struct {
Lang string `json:"lang"`
}
nodeId := c.Param("id")
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
if reqBody.Lang == constants.Nodejs {
if services.IsMasterNode(nodeId) {
_, err := services.InstallNodejsLocalLang()
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
} else {
_, err := services.InstallNodejsRemoteLang(nodeId)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
} else {
HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang))
return
}
// TODO: check if install is successful
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}

17
backend/scripts/install-nodejs.sh Normal file
View File

@@ -0,0 +1,17 @@
#!/bin/env bash
# install nvm
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.2/install.sh | bash
export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
# install Node.js v8.12
nvm install 8.12
# create soft links
ln -s $HOME/.nvm/versions/node/v8.12.0/bin/npm /usr/local/bin/npm
ln -s $HOME/.nvm/versions/node/v8.12.0/bin/node /usr/local/bin/node
# environments manipulation
export NODE_PATH=$HOME.nvm/versions/node/v8.12.0/lib/node_modules
export PATH=$NODE_PATH:$PATH

49
backend/services/node.go
View File

@@ -12,6 +12,7 @@ import (
"encoding/json"
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/gomodule/redigo/redis"
"runtime/debug"
@@ -116,7 +117,7 @@ func handleNodeInfo(key string, data *Data) {
defer s.Close()
var node model.Node
if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
if err := c.Find(bson.M{"key": key}).One(&node); err != nil && err == mgo.ErrNotFound {
// 数据库不存在该节点
node = model.Node{
Key: key,
@@ -133,7 +134,7 @@ func handleNodeInfo(key string, data *Data) {
log.Errorf(err.Error())
return
}
} else {
} else if node.Key != "" {
// 数据库存在该节点
node.Status = constants.StatusOnline
node.UpdateTs = time.Now()
@@ -168,33 +169,27 @@ func UpdateNodeData() {
return
}
//先获取所有Redis的nodekey
list, _ := database.RedisClient.HKeys("nodes")
if i := utils.Contains(list, key); i == false {
// 构造节点数据
data := Data{
Key: key,
Mac: mac,
Ip: ip,
Master: model.IsMaster(),
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
}
// 注册节点到Redis
dataBytes, err := json.Marshal(&data)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil {
log.Errorf(err.Error())
return
}
// 构造节点数据
data := Data{
Key: key,
Mac: mac,
Ip: ip,
Master: model.IsMaster(),
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
}
// 注册节点到Redis
dataBytes, err := json.Marshal(&data)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil {
log.Errorf(err.Error())
return
}
}
func MasterNodeCallback(message redis.Message) (err error) {

231
backend/services/rpc.go Normal file
View File

@@ -0,0 +1,231 @@
package services
import (
"crawlab/constants"
"crawlab/database"
"crawlab/entity"
"crawlab/model"
"crawlab/utils"
"encoding/json"
"fmt"
"github.com/apex/log"
uuid "github.com/satori/go.uuid"
"runtime/debug"
)
type RpcMessage struct {
Id string `json:"id"`
Method string `json:"method"`
Params map[string]string `json:"params"`
Result string `json:"result"`
}
func RpcServerInstallLang(msg RpcMessage) RpcMessage {
lang := GetRpcParam("lang", msg.Params)
if lang == constants.Nodejs {
output, _ := InstallNodejsLocalLang()
msg.Result = output
}
return msg
}
func RpcClientInstallLang(nodeId string, lang string) (output string, err error) {
params := map[string]string{}
params["lang"] = lang
data, err := RpcClientFunc(nodeId, constants.RpcInstallLang, params, 600)()
if err != nil {
return
}
output = data
return
}
func RpcServerInstallDep(msg RpcMessage) RpcMessage {
lang := GetRpcParam("lang", msg.Params)
depName := GetRpcParam("dep_name", msg.Params)
if lang == constants.Python {
output, _ := InstallPythonLocalDep(depName)
msg.Result = output
}
return msg
}
func RpcClientInstallDep(nodeId string, lang string, depName string) (output string, err error) {
params := map[string]string{}
params["lang"] = lang
params["dep_name"] = depName
data, err := RpcClientFunc(nodeId, constants.RpcInstallDep, params, 10)()
if err != nil {
return
}
output = data
return
}
func RpcServerUninstallDep(msg RpcMessage) RpcMessage {
lang := GetRpcParam("lang", msg.Params)
depName := GetRpcParam("dep_name", msg.Params)
if lang == constants.Python {
output, _ := UninstallPythonLocalDep(depName)
msg.Result = output
}
return msg
}
func RpcClientUninstallDep(nodeId string, lang string, depName string) (output string, err error) {
params := map[string]string{}
params["lang"] = lang
params["dep_name"] = depName
data, err := RpcClientFunc(nodeId, constants.RpcUninstallDep, params, 60)()
if err != nil {
return
}
output = data
return
}
func RpcServerGetInstalledDepList(nodeId string, msg RpcMessage) RpcMessage {
lang := GetRpcParam("lang", msg.Params)
if lang == constants.Python {
depList, _ := GetPythonLocalInstalledDepList(nodeId)
resultStr, _ := json.Marshal(depList)
msg.Result = string(resultStr)
} else if lang == constants.Nodejs {
depList, _ := GetNodejsLocalInstalledDepList(nodeId)
resultStr, _ := json.Marshal(depList)
msg.Result = string(resultStr)
}
return msg
}
func RpcClientGetInstalledDepList(nodeId string, lang string) (list []entity.Dependency, err error) {
params := map[string]string{}
params["lang"] = lang
data, err := RpcClientFunc(nodeId, constants.RpcGetInstalledDepList, params, 10)()
if err != nil {
return
}
// 反序列化结果
if err := json.Unmarshal([]byte(data), &list); err != nil {
return list, err
}
return
}
func RpcClientFunc(nodeId string, method string, params map[string]string, timeout int) func() (string, error) {
return func() (result string, err error) {
// 请求ID
id := uuid.NewV4().String()
// 构造RPC消息
msg := RpcMessage{
Id: id,
Method: method,
Params: params,
Result: "",
}
// 发送RPC消息
msgStr := ObjectToString(msg)
if err := database.RedisClient.LPush(fmt.Sprintf("rpc:%s", nodeId), msgStr); err != nil {
return result, err
}
// 获取RPC回复消息
dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", nodeId), timeout)
if err != nil {
return result, err
}
// 反序列化消息
if err := json.Unmarshal([]byte(dataStr), &msg); err != nil {
return result, err
}
return msg.Result, err
}
}
func GetRpcParam(key string, params map[string]string) string {
return params[key]
}
func ObjectToString(params interface{}) string {
bytes, _ := json.Marshal(params)
return utils.BytesToString(bytes)
}
var IsRpcStopped = false
func StopRpcService() {
IsRpcStopped = true
}
func InitRpcService() error {
go func() {
for {
// 获取当前节点
node, err := model.GetCurrentNode()
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
// 获取获取消息队列信息
dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 300)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
// 反序列化消息
var msg RpcMessage
if err := json.Unmarshal([]byte(dataStr), &msg); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
// 根据Method调用本地方法
var replyMsg RpcMessage
if msg.Method == constants.RpcInstallDep {
replyMsg = RpcServerInstallDep(msg)
} else if msg.Method == constants.RpcUninstallDep {
replyMsg = RpcServerUninstallDep(msg)
} else if msg.Method == constants.RpcInstallLang {
replyMsg = RpcServerInstallLang(msg)
} else if msg.Method == constants.RpcGetInstalledDepList {
replyMsg = RpcServerGetInstalledDepList(node.Id.Hex(), msg)
} else {
continue
}
// 发送返回消息
if err := database.RedisClient.LPush(fmt.Sprintf("rpc:%s", node.Id.Hex()), ObjectToString(replyMsg)); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
continue
}
// 如果停止RPC服务，则返回
if IsRpcStopped {
return
}
}
}()
return nil
}

548
backend/services/system.go
View File

@@ -4,28 +4,42 @@ import (
"crawlab/constants"
"crawlab/database"
"crawlab/entity"
"crawlab/lib/cron"
"crawlab/model"
"crawlab/utils"
"encoding/json"
"errors"
"fmt"
"github.com/apex/log"
"github.com/imroc/req"
"os/exec"
"path"
"regexp"
"runtime/debug"
"sort"
"strings"
"sync"
)
// 系统信息 chan 映射
var SystemInfoChanMap = utils.NewChanMap()
func GetRemoteSystemInfo(id string) (sysInfo entity.SystemInfo, err error) {
// 从远端获取系统信息
func GetRemoteSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) {
// 发送消息
msg := entity.NodeMessage{
Type: constants.MsgTypeGetSystemInfo,
NodeId: id,
NodeId: nodeId,
}
// 序列化
msgBytes, _ := json.Marshal(&msg)
if _, err := database.RedisClient.Publish("nodes:"+id, utils.BytesToString(msgBytes)); err != nil {
if _, err := database.RedisClient.Publish("nodes:"+nodeId, utils.BytesToString(msgBytes)); err != nil {
return entity.SystemInfo{}, err
}
// 通道
ch := SystemInfoChanMap.ChanBlocked(id)
ch := SystemInfoChanMap.ChanBlocked(nodeId)
// 等待响应，阻塞
sysInfoStr := <-ch
@@ -38,11 +52,531 @@ func GetRemoteSystemInfo(id string) (sysInfo entity.SystemInfo, err error) {
return sysInfo, nil
}
func GetSystemInfo(id string) (sysInfo entity.SystemInfo, err error) {
if IsMasterNode(id) {
// 获取系统信息
func GetSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) {
if IsMasterNode(nodeId) {
sysInfo, err = model.GetLocalSystemInfo()
} else {
sysInfo, err = GetRemoteSystemInfo(id)
sysInfo, err = GetRemoteSystemInfo(nodeId)
}
return
}
// 获取语言列表
func GetLangList(nodeId string) []entity.Lang {
list := []entity.Lang{
{Name: "Python", ExecutableName: "python", ExecutablePath: "/usr/local/bin/python", DepExecutablePath: "/usr/local/bin/pip"},
{Name: "Node.js", ExecutableName: "node", ExecutablePath: "/usr/local/bin/node", DepExecutablePath: "/usr/local/bin/npm"},
//{Name: "Java", ExecutableName: "java", ExecutablePath: "/usr/local/bin/java"},
}
for i, lang := range list {
list[i].Installed = IsInstalledLang(nodeId, lang)
}
return list
}
// 根据语言名获取语言实例
func GetLangFromLangName(nodeId string, name string) entity.Lang {
langList := GetLangList(nodeId)
for _, lang := range langList {
if lang.ExecutableName == name {
return lang
}
}
return entity.Lang{}
}
// 是否已安装该依赖
func IsInstalledLang(nodeId string, lang entity.Lang) bool {
sysInfo, err := GetSystemInfo(nodeId)
if err != nil {
return false
}
for _, exec := range sysInfo.Executables {
if exec.Path == lang.ExecutablePath {
return true
}
}
return false
}
// 是否已安装该依赖
func IsInstalledDep(installedDepList []entity.Dependency, dep entity.Dependency) bool {
for _, _dep := range installedDepList {
if strings.ToLower(_dep.Name) == strings.ToLower(dep.Name) {
return true
}
}
return false
}
// 初始化函数
func InitDepsFetcher() error {
c := cron.New(cron.WithSeconds())
c.Start()
if _, err := c.AddFunc("0 */5 * * * *", UpdatePythonDepList); err != nil {
return err
}
go func() {
UpdatePythonDepList()
}()
return nil
}
// =========
// Python
// =========
type PythonDepJsonData struct {
Info PythonDepJsonDataInfo `json:"info"`
}
type PythonDepJsonDataInfo struct {
Name string `json:"name"`
Summary string `json:"summary"`
Version string `json:"version"`
}
type PythonDepNameDict struct {
Name string `json:"name"`
Weight int `json:"weight"`
}
type PythonDepNameDictSlice []PythonDepNameDict
func (s PythonDepNameDictSlice) Len() int { return len(s) }
func (s PythonDepNameDictSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s PythonDepNameDictSlice) Less(i, j int) bool { return s[i].Weight > s[j].Weight }
// 获取Python本地依赖列表
func GetPythonDepList(nodeId string, searchDepName string) ([]entity.Dependency, error) {
var list []entity.Dependency
// 先从 Redis 获取
depList, err := GetPythonDepListFromRedis()
if err != nil {
return list, err
}
// 过滤相似的依赖
var depNameList PythonDepNameDictSlice
for _, depName := range depList {
if strings.HasPrefix(strings.ToLower(depName), strings.ToLower(searchDepName)) {
var weight int
if strings.ToLower(depName) == strings.ToLower(searchDepName) {
weight = 3
} else if strings.HasPrefix(strings.ToLower(depName), strings.ToLower(searchDepName)) {
weight = 2
} else {
weight = 1
}
depNameList = append(depNameList, PythonDepNameDict{
Name: depName,
Weight: weight,
})
}
}
// 获取已安装依赖列表
var installedDepList []entity.Dependency
if IsMasterNode(nodeId) {
installedDepList, err = GetPythonLocalInstalledDepList(nodeId)
if err != nil {
return list, err
}
} else {
installedDepList, err = GetPythonRemoteInstalledDepList(nodeId)
if err != nil {
return list, err
}
}
// 根据依赖名排序
sort.Stable(depNameList)
// 遍历依赖名列表，取前20个
for i, depNameDict := range depNameList {
if i > 20 {
break
}
dep := entity.Dependency{
Name: depNameDict.Name,
}
dep.Installed = IsInstalledDep(installedDepList, dep)
list = append(list, dep)
}
// 从依赖源获取信息
//list, err = GetPythonDepListWithInfo(list)
return list, nil
}
// 获取Python依赖的源数据信息
func GetPythonDepListWithInfo(depList []entity.Dependency) ([]entity.Dependency, error) {
var goSync sync.WaitGroup
for i, dep := range depList {
if i > 10 {
break
}
goSync.Add(1)
go func(i int, dep entity.Dependency, depList []entity.Dependency, n *sync.WaitGroup) {
url := fmt.Sprintf("https://pypi.org/pypi/%s/json", dep.Name)
res, err := req.Get(url)
if err != nil {
n.Done()
return
}
var data PythonDepJsonData
if err := res.ToJSON(&data); err != nil {
n.Done()
return
}
depList[i].Version = data.Info.Version
depList[i].Description = data.Info.Summary
n.Done()
}(i, dep, depList, &goSync)
}
goSync.Wait()
return depList, nil
}
func FetchPythonDepInfo(depName string) (entity.Dependency, error) {
url := fmt.Sprintf("https://pypi.org/pypi/%s/json", depName)
res, err := req.Get(url)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return entity.Dependency{}, err
}
var data PythonDepJsonData
if err := res.ToJSON(&data); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return entity.Dependency{}, err
}
dep := entity.Dependency{
Name: depName,
Version: data.Info.Version,
Description: data.Info.Summary,
}
return dep, nil
}
// 从Redis获取Python依赖列表
func GetPythonDepListFromRedis() ([]string, error) {
var list []string
// 从 Redis 获取字符串
rawData, err := database.RedisClient.HGet("system", "deps:python")
if err != nil {
return list, err
}
// 反序列化
if err := json.Unmarshal([]byte(rawData), &list); err != nil {
return list, err
}
// 如果为空，则从依赖源获取列表
if len(list) == 0 {
UpdatePythonDepList()
}
return list, nil
}
// 从Python依赖源获取依赖列表并返回
func FetchPythonDepList() ([]string, error) {
// 依赖URL
url := "https://pypi.tuna.tsinghua.edu.cn/simple"
// 输出列表
var list []string
// 请求URL
res, err := req.Get(url)
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return list, err
}
// 获取响应数据
text, err := res.ToString()
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return list, err
}
// 从响应数据中提取依赖名
regex := regexp.MustCompile("<a href=\".*/\">(.*)</a>")
for _, line := range strings.Split(text, "\n") {
arr := regex.FindStringSubmatch(line)
if len(arr) < 2 {
continue
}
list = append(list, arr[1])
}
// 赋值给列表
return list, nil
}
// 更新Python依赖列表到Redis
func UpdatePythonDepList() {
// 从依赖源获取列表
list, _ := FetchPythonDepList()
// 序列化
listBytes, err := json.Marshal(list)
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return
}
// 设置Redis
if err := database.RedisClient.HSet("system", "deps:python", string(listBytes)); err != nil {
log.Error(err.Error())
debug.PrintStack()
return
}
}
// 获取Python本地已安装的依赖列表
func GetPythonLocalInstalledDepList(nodeId string) ([]entity.Dependency, error) {
var list []entity.Dependency
lang := GetLangFromLangName(nodeId, constants.Python)
if !IsInstalledLang(nodeId, lang) {
return list, errors.New("python is not installed")
}
cmd := exec.Command("pip", "freeze")
outputBytes, err := cmd.Output()
if err != nil {
debug.PrintStack()
return list, err
}
for _, line := range strings.Split(string(outputBytes), "\n") {
arr := strings.Split(line, "==")
if len(arr) < 2 {
continue
}
dep := entity.Dependency{
Name: strings.ToLower(arr[0]),
Version: arr[1],
Installed: true,
}
list = append(list, dep)
}
return list, nil
}
// 获取Python远端依赖列表
func GetPythonRemoteInstalledDepList(nodeId string) ([]entity.Dependency, error) {
depList, err := RpcClientGetInstalledDepList(nodeId, constants.Python)
if err != nil {
return depList, err
}
return depList, nil
}
// 安装Python本地依赖
func InstallPythonLocalDep(depName string) (string, error) {
// 依赖镜像URL
url := "https://pypi.tuna.tsinghua.edu.cn/simple"
cmd := exec.Command("pip", "install", depName, "-i", url)
outputBytes, err := cmd.Output()
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return fmt.Sprintf("error: %s", err.Error()), err
}
return string(outputBytes), nil
}
// 获取Python远端依赖列表
func InstallPythonRemoteDep(nodeId string, depName string) (string, error) {
output, err := RpcClientInstallDep(nodeId, constants.Python, depName)
if err != nil {
return output, err
}
return output, nil
}
// 安装Python本地依赖
func UninstallPythonLocalDep(depName string) (string, error) {
cmd := exec.Command("pip", "uninstall", "-y", depName)
outputBytes, err := cmd.Output()
if err != nil {
log.Errorf(string(outputBytes))
log.Errorf(err.Error())
debug.PrintStack()
return fmt.Sprintf("error: %s", err.Error()), err
}
return string(outputBytes), nil
}
// 获取Python远端依赖列表
func UninstallPythonRemoteDep(nodeId string, depName string) (string, error) {
output, err := RpcClientUninstallDep(nodeId, constants.Python, depName)
if err != nil {
return output, err
}
return output, nil
}
// ==============
// Node.js
// ==============
func InstallNodejsLocalLang() (string, error) {
cmd := exec.Command("/bin/sh", path.Join("scripts", "install-nodejs.sh"))
output, err := cmd.Output()
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return string(output), err
}
// TODO: check if Node.js is installed successfully
return string(output), nil
}
// 获取Node.js远端依赖列表
func InstallNodejsRemoteLang(nodeId string) (string, error) {
output, err := RpcClientInstallLang(nodeId, constants.Nodejs)
if err != nil {
return output, err
}
return output, nil
}
// 获取Nodejs本地已安装的依赖列表
func GetNodejsLocalInstalledDepList(nodeId string) ([]entity.Dependency, error) {
var list []entity.Dependency
lang := GetLangFromLangName(nodeId, constants.Nodejs)
if !IsInstalledLang(nodeId, lang) {
return list, errors.New("nodejs is not installed")
}
cmd := exec.Command("npm", "ls", "-g", "--depth", "0")
outputBytes, _ := cmd.Output()
//if err != nil {
// log.Error("error: " + string(outputBytes))
// debug.PrintStack()
// return list, err
//}
regex := regexp.MustCompile("\\s(.*)@(.*)")
for _, line := range strings.Split(string(outputBytes), "\n") {
arr := regex.FindStringSubmatch(line)
if len(arr) < 3 {
continue
}
dep := entity.Dependency{
Name: strings.ToLower(arr[1]),
Version: arr[2],
Installed: true,
}
list = append(list, dep)
}
return list, nil
}
// 获取Nodejs远端依赖列表
func GetNodejsRemoteInstalledDepList(nodeId string) ([]entity.Dependency, error) {
depList, err := RpcClientGetInstalledDepList(nodeId, constants.Nodejs)
if err != nil {
return depList, err
}
return depList, nil
}
// 安装Nodejs本地依赖
func InstallNodejsLocalDep(depName string) (string, error) {
// 依赖镜像URL
url := "https://registry.npm.taobao.org"
cmd := exec.Command("npm", "install", depName, "-g", "--registry", url)
outputBytes, err := cmd.Output()
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return fmt.Sprintf("error: %s", err.Error()), err
}
return string(outputBytes), nil
}
// 获取Nodejs远端依赖列表
func InstallNodejsRemoteDep(nodeId string, depName string) (string, error) {
output, err := RpcClientInstallDep(nodeId, constants.Nodejs, depName)
if err != nil {
return output, err
}
return output, nil
}
// 安装Nodejs本地依赖
func UninstallNodejsLocalDep(depName string) (string, error) {
cmd := exec.Command("npm", "uninstall", depName, "-g")
outputBytes, err := cmd.Output()
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return fmt.Sprintf("error: %s", err.Error()), err
}
return string(outputBytes), nil
}
// 获取Nodejs远端依赖列表
func UninstallNodejsRemoteDep(nodeId string, depName string) (string, error) {
output, err := RpcClientUninstallDep(nodeId, constants.Nodejs, depName)
if err != nil {
return output, err
}
return output, nil
}
// 获取Nodejs本地依赖列表
func GetNodejsDepList(nodeId string, searchDepName string) (depList []entity.Dependency, err error) {
// 执行shell命令
cmd := exec.Command("npm", "search", "--json", searchDepName)
outputBytes, _ := cmd.Output()
// 获取已安装依赖列表
var installedDepList []entity.Dependency
if IsMasterNode(nodeId) {
installedDepList, err = GetNodejsLocalInstalledDepList(nodeId)
if err != nil {
return depList, err
}
} else {
installedDepList, err = GetNodejsRemoteInstalledDepList(nodeId)
if err != nil {
return depList, err
}
}
// 反序列化
if err := json.Unmarshal(outputBytes, &depList); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return depList, err
}
// 遍历安装列表
for i, dep := range depList {
depList[i].Installed = IsInstalledDep(installedDepList, dep)
}
return depList, nil
}

12
backend/services/task.go
View File

@@ -19,6 +19,7 @@ import (
"runtime"
"runtime/debug"
"strconv"
"strings"
"sync"
"syscall"
"time"
@@ -104,6 +105,17 @@ func AssignTask(task model.Task) error {
// 设置环境变量
func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exec.Cmd {
// 默认把Node.js的全局node_modules加入环境变量
envPath := os.Getenv("PATH")
for _, _path := range strings.Split(envPath, ":") {
if strings.Contains(_path, "/.nvm/versions/node/") {
pathNodeModules := strings.Replace(_path, "/bin", "/lib/node_modules", -1)
_ = os.Setenv("PATH", pathNodeModules+":"+envPath)
_ = os.Setenv("NODE_PATH", pathNodeModules)
break
}
}
// 默认环境变量
cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+taskId)
cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+dataCol)

201
backend/vendor/github.com/imroc/req/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

302
backend/vendor/github.com/imroc/req/README.md generated vendored Normal file
View File

@@ -0,0 +1,302 @@
# req
[![GoDoc](https://godoc.org/github.com/imroc/req?status.svg)](https://godoc.org/github.com/imroc/req)
A golang http request library for humans
Features
========
- Light weight
- Simple
- Easy play with JSON and XML
- Easy for debug and logging
- Easy file uploads and downloads
- Easy manage cookie
- Easy set up proxy
- Easy set timeout
- Easy customize http client
Document
========
[中文](doc/README_cn.md)
Install
=======
``` sh
go get github.com/imroc/req
```
Overview
=======
`req` implements a friendly API over Go's existing `net/http` library.
`Req` and `Resp` are two most important struct, you can think of `Req` as a client that initiate HTTP requests, `Resp` as a information container for the request and response. They all provide simple and convenient APIs that allows you to do a lot of things.
``` go
func (r *Req) Post(url string, v ...interface{}) (*Resp, error)
```
In most cases, only url is required, others are optional, like headers, params, files or body etc.
There is a default `Req` object, all of its' public methods are wrapped by the `req` package, so you can also think of `req` package as a `Req` object
``` go
// use Req object to initiate requests.
r := req.New()
r.Get(url)
// use req package to initiate request.
req.Get(url)
```
You can use `req.New()` to create lots of `*Req` as client with independent configuration
Examples
=======
[Basic](#Basic)
[Set Header](#Set-Header)
[Set Param](#Set-Param)
[Set Body](#Set-Body)
[Debug](#Debug)
[Output Format](#Format)
[ToJSON & ToXML](#ToJSON-ToXML)
[Get *http.Response](#Response)
[Upload](#Upload)
[Download](#Download)
[Cookie](#Cookie)
[Set Timeout](#Set-Timeout)
[Set Proxy](#Set-Proxy)
[Customize Client](#Customize-Client)
## <a name="Basic">Basic</a>
``` go
header := req.Header{
"Accept": "application/json",
"Authorization": "Basic YWRtaW46YWRtaW4=",
}
param := req.Param{
"name": "imroc",
"cmd": "add",
}
// only url is required, others are optional.
r, err = req.Post("http://foo.bar/api", header, param)
if err != nil {
log.Fatal(err)
}
r.ToJSON(&foo) // response => struct/map
log.Printf("%+v", r) // print info (try it, you may surprise)
```
## <a name="Set-Header">Set Header</a>
Use `req.Header` (it is actually a `map[string]string`)
``` go
authHeader := req.Header{
"Accept": "application/json",
"Authorization": "Basic YWRtaW46YWRtaW4=",
}
req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"})
```
use `http.Header`
``` go
header := make(http.Header)
header.Set("Accept", "application/json")
req.Get("https://www.baidu.com", header)
```
## <a name="Set-Param">Set Param</a>
Use `req.Param` (it is actually a `map[string]interface{}`)
``` go
param := req.Param{
"id": "imroc",
"pwd": "roc",
}
req.Get("http://foo.bar/api", param) // http://foo.bar/api?id=imroc&pwd=roc
req.Post(url, param) // body => id=imroc&pwd=roc
```
use `req.QueryParam` force to append params to the url (it is also actually a `map[string]interface{}`)
``` go
req.Post("http://foo.bar/api", req.Param{"name": "roc", "age": "22"}, req.QueryParam{"access_token": "fedledGF9Hg9ehTU"})
/*
POST /api?access_token=fedledGF9Hg9ehTU HTTP/1.1
Host: foo.bar
User-Agent: Go-http-client/1.1
Content-Length: 15
Content-Type: application/x-www-form-urlencoded;charset=UTF-8
Accept-Encoding: gzip
age=22&name=roc
*/
```
## <a name="Set-Body">Set Body</a>
Put `string`, `[]byte` and `io.Reader` as body directly.
``` go
req.Post(url, "id=roc&cmd=query")
```
Put object as xml or json body (add `Content-Type` header automatically)
``` go
req.Post(url, req.BodyJSON(&foo))
req.Post(url, req.BodyXML(&bar))
```
## <a name="Debug">Debug</a>
Set global variable `req.Debug` to true, it will print detail infomation for every request.
``` go
req.Debug = true
req.Post("http://localhost/test" "hi")
```
![post](doc/post.png)
## <a name="Format">Output Format</a>
You can use different kind of output format to log the request and response infomation in your log file in defferent scenarios. For example, use `%+v` output format in the development phase, it allows you to observe the details. Use `%v` or `%-v` output format in production phase, just log the information necessarily.
### `%+v` or `%+s`
Output in detail
``` go
r, _ := req.Post(url, header, param)
log.Printf("%+v", r) // output the same format as Debug is enabled
```
### `%v` or `%s`
Output in simple way (default format)
``` go
r, _ := req.Get(url, param)
log.Printf("%v\n", r) // GET http://foo.bar/api?name=roc&cmd=add {"code":"0","msg":"success"}
log.Prinln(r) // smae as above
```
### `%-v` or `%-s`
Output in simple way and keep all in one line (request body or response body may have multiple lines, this format will replace `"\r"` or `"\n"` with `" "`, it's useful when doing some search in your log file)
### Flag
You can call `SetFlags` to control the output content, decide which pieces can be output.
``` go
const (
LreqHead = 1 << iota // output request head (request line and request header)
LreqBody // output request body
LrespHead // output response head (response line and response header)
LrespBody // output response body
Lcost // output time costed by the request
LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody
)
```
``` go
req.SetFlags(req.LreqHead | req.LreqBody | req.LrespHead)
```
### Monitoring time consuming
``` go
req.SetFlags(req.LstdFlags | req.Lcost) // output format add time costed by request
r,_ := req.Get(url)
log.Println(r) // http://foo.bar/api 3.260802ms {"code":0 "msg":"success"}
if r.Cost() > 3 * time.Second { // check cost
log.Println("WARN: slow request:", r)
}
```
## <a name="ToJSON-ToXML">ToJSON & ToXML</a>
``` go
r, _ := req.Get(url)
r.ToJSON(&foo)
r, _ = req.Post(url, req.BodyXML(&bar))
r.ToXML(&baz)
```
## <a name="Response">Get *http.Response</a>
```go
// func (r *Req) Response() *http.Response
r, _ := req.Get(url)
resp := r.Response()
fmt.Println(resp.StatusCode)
```
## <a name="Upload">Upload</a>
Use `req.File` to match files
``` go
req.Post(url, req.File("imroc.png"), req.File("/Users/roc/Pictures/*.png"))
```
Use `req.FileUpload` to fully control
``` go
file, _ := os.Open("imroc.png")
req.Post(url, req.FileUpload{
File: file,
FieldName: "file", // FieldName is form field name
FileName: "avatar.png", //Filename is the name of the file that you wish to upload. We use this to guess the mimetype as well as pass it onto the server
})
```
Use `req.UploadProgress` to listen upload progress
```go
progress := func(current, total int64) {
fmt.Println(float32(current)/float32(total)*100, "%")
}
req.Post(url, req.File("/Users/roc/Pictures/*.png"), req.UploadProgress(progress))
fmt.Println("upload complete")
```
## <a name="Download">Download</a>
``` go
r, _ := req.Get(url)
r.ToFile("imroc.png")
```
Use `req.DownloadProgress` to listen download progress
```go
progress := func(current, total int64) {
fmt.Println(float32(current)/float32(total)*100, "%")
}
r, _ := req.Get(url, req.DownloadProgress(progress))
r.ToFile("hello.mp4")
fmt.Println("download complete")
```
## <a name="Cookie">Cookie</a>
By default, the underlying `*http.Client` will manage your cookie(send cookie header to server automatically if server has set a cookie for you), you can disable it by calling this function :
``` go
req.EnableCookie(false)
```
and you can set cookie in request just using `*http.Cookie`
``` go
cookie := new(http.Cookie)
// ......
req.Get(url, cookie)
```
## <a name="Set-Timeout">Set Timeout</a>
``` go
req.SetTimeout(50 * time.Second)
```
## <a name="Set-Proxy">Set Proxy</a>
By default, req use proxy from system environment if `http_proxy` or `https_proxy` is specified, you can set a custom proxy or disable it by set `nil`
``` go
req.SetProxy(func(r *http.Request) (*url.URL, error) {
if strings.Contains(r.URL.Hostname(), "google") {
return url.Parse("http://my.vpn.com:23456")
}
return nil, nil
})
```
Set a simple proxy (use fixed proxy url for every request)
``` go
req.SetProxyUrl("http://my.proxy.com:23456")
```
## <a name="Customize-Client">Customize Client</a>
Use `SetClient` to change the default underlying `*http.Client`
``` go
req.SetClient(client)
```
Specify independent http client for some requests
``` go
client := &http.Client{Timeout: 30 * time.Second}
req.Get(url, client)
```
Change some properties of default client you want
``` go
req.Client().Jar, _ = cookiejar.New(nil)
trans, _ := req.Client().Transport.(*http.Transport)
trans.MaxIdleConns = 20
trans.TLSHandshakeTimeout = 20 * time.Second
trans.DisableKeepAlives = true
trans.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
```

216
backend/vendor/github.com/imroc/req/dump.go generated vendored Normal file
View File

@@ -0,0 +1,216 @@
package req
import (
"bufio"
"bytes"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/http/httputil"
"net/url"
"strings"
"time"
)
// Debug enable debug mode if set to true
var Debug bool
// dumpConn is a net.Conn which writes to Writer and reads from Reader
type dumpConn struct {
io.Writer
io.Reader
}
func (c *dumpConn) Close() error { return nil }
func (c *dumpConn) LocalAddr() net.Addr { return nil }
func (c *dumpConn) RemoteAddr() net.Addr { return nil }
func (c *dumpConn) SetDeadline(t time.Time) error { return nil }
func (c *dumpConn) SetReadDeadline(t time.Time) error { return nil }
func (c *dumpConn) SetWriteDeadline(t time.Time) error { return nil }
// delegateReader is a reader that delegates to another reader,
// once it arrives on a channel.
type delegateReader struct {
c chan io.Reader
r io.Reader // nil until received from c
}
func (r *delegateReader) Read(p []byte) (int, error) {
if r.r == nil {
r.r = <-r.c
}
return r.r.Read(p)
}
type dummyBody struct {
N int
off int
}
func (d *dummyBody) Read(p []byte) (n int, err error) {
if d.N <= 0 {
err = io.EOF
return
}
left := d.N - d.off
if left <= 0 {
err = io.EOF
return
}
if l := len(p); l > 0 {
if l >= left {
n = left
err = io.EOF
} else {
n = l
}
d.off += n
for i := 0; i < n; i++ {
p[i] = '*'
}
}
return
}
func (d *dummyBody) Close() error {
return nil
}
type dumpBuffer struct {
bytes.Buffer
}
func (b *dumpBuffer) Write(p []byte) {
if b.Len() > 0 {
b.Buffer.WriteString("\r\n\r\n")
}
b.Buffer.Write(p)
}
func (b *dumpBuffer) WriteString(s string) {
b.Write([]byte(s))
}
func (r *Resp) dumpRequest(dump *dumpBuffer) {
head := r.r.flag&LreqHead != 0
body := r.r.flag&LreqBody != 0
if head {
r.dumpReqHead(dump)
}
if body {
if r.multipartHelper != nil {
dump.Write(r.multipartHelper.Dump())
} else if len(r.reqBody) > 0 {
dump.Write(r.reqBody)
}
}
}
func (r *Resp) dumpReqHead(dump *dumpBuffer) {
reqSend := new(http.Request)
*reqSend = *r.req
if reqSend.URL.Scheme == "https" {
reqSend.URL = new(url.URL)
*reqSend.URL = *r.req.URL
reqSend.URL.Scheme = "http"
}
if reqSend.ContentLength > 0 {
reqSend.Body = &dummyBody{N: int(reqSend.ContentLength)}
} else {
reqSend.Body = &dummyBody{N: 1}
}
// Use the actual Transport code to record what we would send
// on the wire, but not using TCP. Use a Transport with a
// custom dialer that returns a fake net.Conn that waits
// for the full input (and recording it), and then responds
// with a dummy response.
var buf bytes.Buffer // records the output
pr, pw := io.Pipe()
defer pw.Close()
dr := &delegateReader{c: make(chan io.Reader)}
t := &http.Transport{
Dial: func(net, addr string) (net.Conn, error) {
return &dumpConn{io.MultiWriter(&buf, pw), dr}, nil
},
}
defer t.CloseIdleConnections()
client := new(http.Client)
*client = *r.client
client.Transport = t
// Wait for the request before replying with a dummy response:
go func() {
req, err := http.ReadRequest(bufio.NewReader(pr))
if err == nil {
// Ensure all the body is read; otherwise
// we'll get a partial dump.
io.Copy(ioutil.Discard, req.Body)
req.Body.Close()
}
dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n")
pr.Close()
}()
_, err := client.Do(reqSend)
if err != nil {
dump.WriteString(err.Error())
} else {
reqDump := buf.Bytes()
if i := bytes.Index(reqDump, []byte("\r\n\r\n")); i >= 0 {
reqDump = reqDump[:i]
}
dump.Write(reqDump)
}
}
func (r *Resp) dumpResponse(dump *dumpBuffer) {
head := r.r.flag&LrespHead != 0
body := r.r.flag&LrespBody != 0
if head {
respDump, err := httputil.DumpResponse(r.resp, false)
if err != nil {
dump.WriteString(err.Error())
} else {
if i := bytes.Index(respDump, []byte("\r\n\r\n")); i >= 0 {
respDump = respDump[:i]
}
dump.Write(respDump)
}
}
if body && len(r.Bytes()) > 0 {
dump.Write(r.Bytes())
}
}
// Cost return the time cost of the request
func (r *Resp) Cost() time.Duration {
return r.cost
}
// Dump dump the request
func (r *Resp) Dump() string {
dump := new(dumpBuffer)
if r.r.flag&Lcost != 0 {
dump.WriteString(fmt.Sprint(r.cost))
}
r.dumpRequest(dump)
l := dump.Len()
if l > 0 {
dump.WriteString("=================================")
l = dump.Len()
}
r.dumpResponse(dump)
return dump.String()
}

688
backend/vendor/github.com/imroc/req/req.go generated vendored Normal file
View File

@@ -0,0 +1,688 @@
package req
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
// default *Req
var std = New()
// flags to decide which part can be outputed
const (
LreqHead = 1 << iota // output request head (request line and request header)
LreqBody // output request body
LrespHead // output response head (response line and response header)
LrespBody // output response body
Lcost // output time costed by the request
LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody
)
// Header represents http request header
type Header map[string]string
func (h Header) Clone() Header {
if h == nil {
return nil
}
hh := Header{}
for k, v := range h {
hh[k] = v
}
return hh
}
// Param represents http request param
type Param map[string]interface{}
// QueryParam is used to force append http request param to the uri
type QueryParam map[string]interface{}
// Host is used for set request's Host
type Host string
// FileUpload represents a file to upload
type FileUpload struct {
// filename in multipart form.
FileName string
// form field name
FieldName string
// file to uplaod, required
File io.ReadCloser
}
type DownloadProgress func(current, total int64)
type UploadProgress func(current, total int64)
// File upload files matching the name pattern such as
// /usr/*/bin/go* (assuming the Separator is '/')
func File(patterns ...string) interface{} {
matches := []string{}
for _, pattern := range patterns {
m, err := filepath.Glob(pattern)
if err != nil {
return err
}
matches = append(matches, m...)
}
if len(matches) == 0 {
return errors.New("req: no file have been matched")
}
uploads := []FileUpload{}
for _, match := range matches {
if s, e := os.Stat(match); e != nil || s.IsDir() {
continue
}
file, _ := os.Open(match)
uploads = append(uploads, FileUpload{
File: file,
FileName: filepath.Base(match),
FieldName: "media",
})
}
return uploads
}
type bodyJson struct {
v interface{}
}
type bodyXml struct {
v interface{}
}
// BodyJSON make the object be encoded in json format and set it to the request body
func BodyJSON(v interface{}) *bodyJson {
return &bodyJson{v: v}
}
// BodyXML make the object be encoded in xml format and set it to the request body
func BodyXML(v interface{}) *bodyXml {
return &bodyXml{v: v}
}
// Req is a convenient client for initiating requests
type Req struct {
client *http.Client
jsonEncOpts *jsonEncOpts
xmlEncOpts *xmlEncOpts
flag int
}
// New create a new *Req
func New() *Req {
return &Req{flag: LstdFlags}
}
type param struct {
url.Values
}
func (p *param) getValues() url.Values {
if p.Values == nil {
p.Values = make(url.Values)
}
return p.Values
}
func (p *param) Copy(pp param) {
if pp.Values == nil {
return
}
vs := p.getValues()
for key, values := range pp.Values {
for _, value := range values {
vs.Add(key, value)
}
}
}
func (p *param) Adds(m map[string]interface{}) {
if len(m) == 0 {
return
}
vs := p.getValues()
for k, v := range m {
vs.Add(k, fmt.Sprint(v))
}
}
func (p *param) Empty() bool {
return p.Values == nil
}
// Do execute a http request with sepecify method and url,
// and it can also have some optional params, depending on your needs.
func (r *Req) Do(method, rawurl string, vs ...interface{}) (resp *Resp, err error) {
if rawurl == "" {
return nil, errors.New("req: url not specified")
}
req := &http.Request{
Method: method,
Header: make(http.Header),
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
}
resp = &Resp{req: req, r: r}
var queryParam param
var formParam param
var uploads []FileUpload
var uploadProgress UploadProgress
var progress func(int64, int64)
var delayedFunc []func()
var lastFunc []func()
for _, v := range vs {
switch vv := v.(type) {
case Header:
for key, value := range vv {
req.Header.Add(key, value)
}
case http.Header:
for key, values := range vv {
for _, value := range values {
req.Header.Add(key, value)
}
}
case *bodyJson:
fn, err := setBodyJson(req, resp, r.jsonEncOpts, vv.v)
if err != nil {
return nil, err
}
delayedFunc = append(delayedFunc, fn)
case *bodyXml:
fn, err := setBodyXml(req, resp, r.xmlEncOpts, vv.v)
if err != nil {
return nil, err
}
delayedFunc = append(delayedFunc, fn)
case url.Values:
p := param{vv}
if method == "GET" || method == "HEAD" {
queryParam.Copy(p)
} else {
formParam.Copy(p)
}
case Param:
if method == "GET" || method == "HEAD" {
queryParam.Adds(vv)
} else {
formParam.Adds(vv)
}
case QueryParam:
queryParam.Adds(vv)
case string:
setBodyBytes(req, resp, []byte(vv))
case []byte:
setBodyBytes(req, resp, vv)
case bytes.Buffer:
setBodyBytes(req, resp, vv.Bytes())
case *http.Client:
resp.client = vv
case FileUpload:
uploads = append(uploads, vv)
case []FileUpload:
uploads = append(uploads, vv...)
case *http.Cookie:
req.AddCookie(vv)
case Host:
req.Host = string(vv)
case io.Reader:
fn := setBodyReader(req, resp, vv)
lastFunc = append(lastFunc, fn)
case UploadProgress:
uploadProgress = vv
case DownloadProgress:
resp.downloadProgress = vv
case func(int64, int64):
progress = vv
case context.Context:
req = req.WithContext(vv)
resp.req = req
case error:
return nil, vv
}
}
if length := req.Header.Get("Content-Length"); length != "" {
if l, err := strconv.ParseInt(length, 10, 64); err == nil {
req.ContentLength = l
}
}
if len(uploads) > 0 && (req.Method == "POST" || req.Method == "PUT") { // multipart
var up UploadProgress
if uploadProgress != nil {
up = uploadProgress
} else if progress != nil {
up = UploadProgress(progress)
}
multipartHelper := &multipartHelper{
form: formParam.Values,
uploads: uploads,
uploadProgress: up,
}
multipartHelper.Upload(req)
resp.multipartHelper = multipartHelper
} else {
if progress != nil {
resp.downloadProgress = DownloadProgress(progress)
}
if !formParam.Empty() {
if req.Body != nil {
queryParam.Copy(formParam)
} else {
setBodyBytes(req, resp, []byte(formParam.Encode()))
setContentType(req, "application/x-www-form-urlencoded; charset=UTF-8")
}
}
}
if !queryParam.Empty() {
paramStr := queryParam.Encode()
if strings.IndexByte(rawurl, '?') == -1 {
rawurl = rawurl + "?" + paramStr
} else {
rawurl = rawurl + "&" + paramStr
}
}
u, err := url.Parse(rawurl)
if err != nil {
return nil, err
}
req.URL = u
if host := req.Header.Get("Host"); host != "" {
req.Host = host
}
for _, fn := range delayedFunc {
fn()
}
if resp.client == nil {
resp.client = r.Client()
}
var response *http.Response
if r.flag&Lcost != 0 {
before := time.Now()
response, err = resp.client.Do(req)
after := time.Now()
resp.cost = after.Sub(before)
} else {
response, err = resp.client.Do(req)
}
if err != nil {
return nil, err
}
for _, fn := range lastFunc {
fn()
}
resp.resp = response
if _, ok := resp.client.Transport.(*http.Transport); ok && response.Header.Get("Content-Encoding") == "gzip" && req.Header.Get("Accept-Encoding") != "" {
body, err := gzip.NewReader(response.Body)
if err != nil {
return nil, err
}
response.Body = body
}
// output detail if Debug is enabled
if Debug {
fmt.Println(resp.Dump())
}
return
}
func setBodyBytes(req *http.Request, resp *Resp, data []byte) {
resp.reqBody = data
req.Body = ioutil.NopCloser(bytes.NewReader(data))
req.ContentLength = int64(len(data))
}
func setBodyJson(req *http.Request, resp *Resp, opts *jsonEncOpts, v interface{}) (func(), error) {
var data []byte
switch vv := v.(type) {
case string:
data = []byte(vv)
case []byte:
data = vv
case *bytes.Buffer:
data = vv.Bytes()
default:
if opts != nil {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetIndent(opts.indentPrefix, opts.indentValue)
enc.SetEscapeHTML(opts.escapeHTML)
err := enc.Encode(v)
if err != nil {
return nil, err
}
data = buf.Bytes()
} else {
var err error
data, err = json.Marshal(v)
if err != nil {
return nil, err
}
}
}
setBodyBytes(req, resp, data)
delayedFunc := func() {
setContentType(req, "application/json; charset=UTF-8")
}
return delayedFunc, nil
}
func setBodyXml(req *http.Request, resp *Resp, opts *xmlEncOpts, v interface{}) (func(), error) {
var data []byte
switch vv := v.(type) {
case string:
data = []byte(vv)
case []byte:
data = vv
case *bytes.Buffer:
data = vv.Bytes()
default:
if opts != nil {
var buf bytes.Buffer
enc := xml.NewEncoder(&buf)
enc.Indent(opts.prefix, opts.indent)
err := enc.Encode(v)
if err != nil {
return nil, err
}
data = buf.Bytes()
} else {
var err error
data, err = xml.Marshal(v)
if err != nil {
return nil, err
}
}
}
setBodyBytes(req, resp, data)
delayedFunc := func() {
setContentType(req, "application/xml; charset=UTF-8")
}
return delayedFunc, nil
}
func setContentType(req *http.Request, contentType string) {
if req.Header.Get("Content-Type") == "" {
req.Header.Set("Content-Type", contentType)
}
}
func setBodyReader(req *http.Request, resp *Resp, rd io.Reader) func() {
var rc io.ReadCloser
switch r := rd.(type) {
case *os.File:
stat, err := r.Stat()
if err == nil {
req.ContentLength = stat.Size()
}
rc = r
case io.ReadCloser:
rc = r
default:
rc = ioutil.NopCloser(rd)
}
bw := &bodyWrapper{
ReadCloser: rc,
limit: 102400,
}
req.Body = bw
lastFunc := func() {
resp.reqBody = bw.buf.Bytes()
}
return lastFunc
}
type bodyWrapper struct {
io.ReadCloser
buf bytes.Buffer
limit int
}
func (b *bodyWrapper) Read(p []byte) (n int, err error) {
n, err = b.ReadCloser.Read(p)
if left := b.limit - b.buf.Len(); left > 0 && n > 0 {
if n <= left {
b.buf.Write(p[:n])
} else {
b.buf.Write(p[:left])
}
}
return
}
type multipartHelper struct {
form url.Values
uploads []FileUpload
dump []byte
uploadProgress UploadProgress
}
func (m *multipartHelper) Upload(req *http.Request) {
pr, pw := io.Pipe()
bodyWriter := multipart.NewWriter(pw)
go func() {
for key, values := range m.form {
for _, value := range values {
bodyWriter.WriteField(key, value)
}
}
var upload func(io.Writer, io.Reader) error
if m.uploadProgress != nil {
var total int64
for _, up := range m.uploads {
if file, ok := up.File.(*os.File); ok {
stat, err := file.Stat()
if err != nil {
continue
}
total += stat.Size()
}
}
var current int64
buf := make([]byte, 1024)
var lastTime time.Time
upload = func(w io.Writer, r io.Reader) error {
for {
n, err := r.Read(buf)
if n > 0 {
_, _err := w.Write(buf[:n])
if _err != nil {
return _err
}
current += int64(n)
if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond {
lastTime = now
m.uploadProgress(current, total)
}
}
if err == io.EOF {
return nil
}
if err != nil {
return err
}
}
}
}
i := 0
for _, up := range m.uploads {
if up.FieldName == "" {
i++
up.FieldName = "file" + strconv.Itoa(i)
}
fileWriter, err := bodyWriter.CreateFormFile(up.FieldName, up.FileName)
if err != nil {
continue
}
//iocopy
if upload == nil {
io.Copy(fileWriter, up.File)
} else {
if _, ok := up.File.(*os.File); ok {
upload(fileWriter, up.File)
} else {
io.Copy(fileWriter, up.File)
}
}
up.File.Close()
}
bodyWriter.Close()
pw.Close()
}()
req.Header.Set("Content-Type", bodyWriter.FormDataContentType())
req.Body = ioutil.NopCloser(pr)
}
func (m *multipartHelper) Dump() []byte {
if m.dump != nil {
return m.dump
}
var buf bytes.Buffer
bodyWriter := multipart.NewWriter(&buf)
for key, values := range m.form {
for _, value := range values {
m.writeField(bodyWriter, key, value)
}
}
for _, up := range m.uploads {
m.writeFile(bodyWriter, up.FieldName, up.FileName)
}
bodyWriter.Close()
m.dump = buf.Bytes()
return m.dump
}
func (m *multipartHelper) writeField(w *multipart.Writer, fieldname, value string) error {
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition",
fmt.Sprintf(`form-data; name="%s"`, fieldname))
p, err := w.CreatePart(h)
if err != nil {
return err
}
_, err = p.Write([]byte(value))
return err
}
func (m *multipartHelper) writeFile(w *multipart.Writer, fieldname, filename string) error {
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition",
fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
fieldname, filename))
h.Set("Content-Type", "application/octet-stream")
p, err := w.CreatePart(h)
if err != nil {
return err
}
_, err = p.Write([]byte("******"))
return err
}
// Get execute a http GET request
func (r *Req) Get(url string, v ...interface{}) (*Resp, error) {
return r.Do("GET", url, v...)
}
// Post execute a http POST request
func (r *Req) Post(url string, v ...interface{}) (*Resp, error) {
return r.Do("POST", url, v...)
}
// Put execute a http PUT request
func (r *Req) Put(url string, v ...interface{}) (*Resp, error) {
return r.Do("PUT", url, v...)
}
// Patch execute a http PATCH request
func (r *Req) Patch(url string, v ...interface{}) (*Resp, error) {
return r.Do("PATCH", url, v...)
}
// Delete execute a http DELETE request
func (r *Req) Delete(url string, v ...interface{}) (*Resp, error) {
return r.Do("DELETE", url, v...)
}
// Head execute a http HEAD request
func (r *Req) Head(url string, v ...interface{}) (*Resp, error) {
return r.Do("HEAD", url, v...)
}
// Options execute a http OPTIONS request
func (r *Req) Options(url string, v ...interface{}) (*Resp, error) {
return r.Do("OPTIONS", url, v...)
}
// Get execute a http GET request
func Get(url string, v ...interface{}) (*Resp, error) {
return std.Get(url, v...)
}
// Post execute a http POST request
func Post(url string, v ...interface{}) (*Resp, error) {
return std.Post(url, v...)
}
// Put execute a http PUT request
func Put(url string, v ...interface{}) (*Resp, error) {
return std.Put(url, v...)
}
// Head execute a http HEAD request
func Head(url string, v ...interface{}) (*Resp, error) {
return std.Head(url, v...)
}
// Options execute a http OPTIONS request
func Options(url string, v ...interface{}) (*Resp, error) {
return std.Options(url, v...)
}
// Delete execute a http DELETE request
func Delete(url string, v ...interface{}) (*Resp, error) {
return std.Delete(url, v...)
}
// Patch execute a http PATCH request
func Patch(url string, v ...interface{}) (*Resp, error) {
return std.Patch(url, v...)
}
// Do execute request.
func Do(method, url string, v ...interface{}) (*Resp, error) {
return std.Do(method, url, v...)
}

215
backend/vendor/github.com/imroc/req/resp.go generated vendored Normal file
View File

@@ -0,0 +1,215 @@
package req
import (
"encoding/json"
"encoding/xml"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"regexp"
"time"
)
// Resp represents a request with it's response
type Resp struct {
r *Req
req *http.Request
resp *http.Response
client *http.Client
cost time.Duration
*multipartHelper
reqBody []byte
respBody []byte
downloadProgress DownloadProgress
err error // delayed error
}
// Request returns *http.Request
func (r *Resp) Request() *http.Request {
return r.req
}
// Response returns *http.Response
func (r *Resp) Response() *http.Response {
return r.resp
}
// Bytes returns response body as []byte
func (r *Resp) Bytes() []byte {
data, _ := r.ToBytes()
return data
}
// ToBytes returns response body as []byte,
// return error if error happend when reading
// the response body
func (r *Resp) ToBytes() ([]byte, error) {
if r.err != nil {
return nil, r.err
}
if r.respBody != nil {
return r.respBody, nil
}
defer r.resp.Body.Close()
respBody, err := ioutil.ReadAll(r.resp.Body)
if err != nil {
r.err = err
return nil, err
}
r.respBody = respBody
return r.respBody, nil
}
// String returns response body as string
func (r *Resp) String() string {
data, _ := r.ToBytes()
return string(data)
}
// ToString returns response body as string,
// return error if error happend when reading
// the response body
func (r *Resp) ToString() (string, error) {
data, err := r.ToBytes()
return string(data), err
}
// ToJSON convert json response body to struct or map
func (r *Resp) ToJSON(v interface{}) error {
data, err := r.ToBytes()
if err != nil {
return err
}
return json.Unmarshal(data, v)
}
// ToXML convert xml response body to struct or map
func (r *Resp) ToXML(v interface{}) error {
data, err := r.ToBytes()
if err != nil {
return err
}
return xml.Unmarshal(data, v)
}
// ToFile download the response body to file with optional download callback
func (r *Resp) ToFile(name string) error {
//TODO set name to the suffix of url path if name == ""
file, err := os.Create(name)
if err != nil {
return err
}
defer file.Close()
if r.respBody != nil {
_, err = file.Write(r.respBody)
return err
}
if r.downloadProgress != nil && r.resp.ContentLength > 0 {
return r.download(file)
}
defer r.resp.Body.Close()
_, err = io.Copy(file, r.resp.Body)
return err
}
func (r *Resp) download(file *os.File) error {
p := make([]byte, 1024)
b := r.resp.Body
defer b.Close()
total := r.resp.ContentLength
var current int64
var lastTime time.Time
for {
l, err := b.Read(p)
if l > 0 {
_, _err := file.Write(p[:l])
if _err != nil {
return _err
}
current += int64(l)
if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond {
lastTime = now
r.downloadProgress(current, total)
}
}
if err != nil {
if err == io.EOF {
return nil
}
return err
}
}
}
var regNewline = regexp.MustCompile(`\n|\r`)
func (r *Resp) autoFormat(s fmt.State) {
req := r.req
if r.r.flag&Lcost != 0 {
fmt.Fprint(s, req.Method, " ", req.URL.String(), " ", r.cost)
} else {
fmt.Fprint(s, req.Method, " ", req.URL.String())
}
// test if it is should be outputed pretty
var pretty bool
var parts []string
addPart := func(part string) {
if part == "" {
return
}
parts = append(parts, part)
if !pretty && regNewline.MatchString(part) {
pretty = true
}
}
if r.r.flag&LreqBody != 0 { // request body
addPart(string(r.reqBody))
}
if r.r.flag&LrespBody != 0 { // response body
addPart(r.String())
}
for _, part := range parts {
if pretty {
fmt.Fprint(s, "\n")
}
fmt.Fprint(s, " ", part)
}
}
func (r *Resp) miniFormat(s fmt.State) {
req := r.req
if r.r.flag&Lcost != 0 {
fmt.Fprint(s, req.Method, " ", req.URL.String(), " ", r.cost)
} else {
fmt.Fprint(s, req.Method, " ", req.URL.String())
}
if r.r.flag&LreqBody != 0 && len(r.reqBody) > 0 { // request body
str := regNewline.ReplaceAllString(string(r.reqBody), " ")
fmt.Fprint(s, " ", str)
}
if r.r.flag&LrespBody != 0 && r.String() != "" { // response body
str := regNewline.ReplaceAllString(r.String(), " ")
fmt.Fprint(s, " ", str)
}
}
// Format fort the response
func (r *Resp) Format(s fmt.State, verb rune) {
if r == nil || r.req == nil {
return
}
if s.Flag('+') { // include header and format pretty.
fmt.Fprint(s, r.Dump())
} else if s.Flag('-') { // keep all informations in one line.
r.miniFormat(s)
} else { // auto
r.autoFormat(s)
}
}

236
backend/vendor/github.com/imroc/req/setting.go generated vendored Normal file
View File

@@ -0,0 +1,236 @@
package req
import (
"crypto/tls"
"errors"
"net"
"net/http"
"net/http/cookiejar"
"net/url"
"time"
)
// create a default client
func newClient() *http.Client {
jar, _ := cookiejar.New(nil)
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
return &http.Client{
Jar: jar,
Transport: transport,
Timeout: 2 * time.Minute,
}
}
// Client return the default underlying http client
func (r *Req) Client() *http.Client {
if r.client == nil {
r.client = newClient()
}
return r.client
}
// Client return the default underlying http client
func Client() *http.Client {
return std.Client()
}
// SetClient sets the underlying http.Client.
func (r *Req) SetClient(client *http.Client) {
r.client = client // use default if client == nil
}
// SetClient sets the default http.Client for requests.
func SetClient(client *http.Client) {
std.SetClient(client)
}
// SetFlags control display format of *Resp
func (r *Req) SetFlags(flags int) {
r.flag = flags
}
// SetFlags control display format of *Resp
func SetFlags(flags int) {
std.SetFlags(flags)
}
// Flags return output format for the *Resp
func (r *Req) Flags() int {
return r.flag
}
// Flags return output format for the *Resp
func Flags() int {
return std.Flags()
}
func (r *Req) getTransport() *http.Transport {
trans, _ := r.Client().Transport.(*http.Transport)
return trans
}
// EnableInsecureTLS allows insecure https
func (r *Req) EnableInsecureTLS(enable bool) {
trans := r.getTransport()
if trans == nil {
return
}
if trans.TLSClientConfig == nil {
trans.TLSClientConfig = &tls.Config{}
}
trans.TLSClientConfig.InsecureSkipVerify = enable
}
func EnableInsecureTLS(enable bool) {
std.EnableInsecureTLS(enable)
}
// EnableCookieenable or disable cookie manager
func (r *Req) EnableCookie(enable bool) {
if enable {
jar, _ := cookiejar.New(nil)
r.Client().Jar = jar
} else {
r.Client().Jar = nil
}
}
// EnableCookieenable or disable cookie manager
func EnableCookie(enable bool) {
std.EnableCookie(enable)
}
// SetTimeout sets the timeout for every request
func (r *Req) SetTimeout(d time.Duration) {
r.Client().Timeout = d
}
// SetTimeout sets the timeout for every request
func SetTimeout(d time.Duration) {
std.SetTimeout(d)
}
// SetProxyUrl set the simple proxy with fixed proxy url
func (r *Req) SetProxyUrl(rawurl string) error {
trans := r.getTransport()
if trans == nil {
return errors.New("req: no transport")
}
u, err := url.Parse(rawurl)
if err != nil {
return err
}
trans.Proxy = http.ProxyURL(u)
return nil
}
// SetProxyUrl set the simple proxy with fixed proxy url
func SetProxyUrl(rawurl string) error {
return std.SetProxyUrl(rawurl)
}
// SetProxy sets the proxy for every request
func (r *Req) SetProxy(proxy func(*http.Request) (*url.URL, error)) error {
trans := r.getTransport()
if trans == nil {
return errors.New("req: no transport")
}
trans.Proxy = proxy
return nil
}
// SetProxy sets the proxy for every request
func SetProxy(proxy func(*http.Request) (*url.URL, error)) error {
return std.SetProxy(proxy)
}
type jsonEncOpts struct {
indentPrefix string
indentValue string
escapeHTML bool
}
func (r *Req) getJSONEncOpts() *jsonEncOpts {
if r.jsonEncOpts == nil {
r.jsonEncOpts = &jsonEncOpts{escapeHTML: true}
}
return r.jsonEncOpts
}
// SetJSONEscapeHTML specifies whether problematic HTML characters
// should be escaped inside JSON quoted strings.
// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e
// to avoid certain safety problems that can arise when embedding JSON in HTML.
//
// In non-HTML settings where the escaping interferes with the readability
// of the output, SetEscapeHTML(false) disables this behavior.
func (r *Req) SetJSONEscapeHTML(escape bool) {
opts := r.getJSONEncOpts()
opts.escapeHTML = escape
}
// SetJSONEscapeHTML specifies whether problematic HTML characters
// should be escaped inside JSON quoted strings.
// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e
// to avoid certain safety problems that can arise when embedding JSON in HTML.
//
// In non-HTML settings where the escaping interferes with the readability
// of the output, SetEscapeHTML(false) disables this behavior.
func SetJSONEscapeHTML(escape bool) {
std.SetJSONEscapeHTML(escape)
}
// SetJSONIndent instructs the encoder to format each subsequent encoded
// value as if indented by the package-level function Indent(dst, src, prefix, indent).
// Calling SetIndent("", "") disables indentation.
func (r *Req) SetJSONIndent(prefix, indent string) {
opts := r.getJSONEncOpts()
opts.indentPrefix = prefix
opts.indentValue = indent
}
// SetJSONIndent instructs the encoder to format each subsequent encoded
// value as if indented by the package-level function Indent(dst, src, prefix, indent).
// Calling SetIndent("", "") disables indentation.
func SetJSONIndent(prefix, indent string) {
std.SetJSONIndent(prefix, indent)
}
type xmlEncOpts struct {
prefix string
indent string
}
func (r *Req) getXMLEncOpts() *xmlEncOpts {
if r.xmlEncOpts == nil {
r.xmlEncOpts = &xmlEncOpts{}
}
return r.xmlEncOpts
}
// SetXMLIndent sets the encoder to generate XML in which each element
// begins on a new indented line that starts with prefix and is followed by
// one or more copies of indent according to the nesting depth.
func (r *Req) SetXMLIndent(prefix, indent string) {
opts := r.getXMLEncOpts()
opts.prefix = prefix
opts.indent = indent
}
// SetXMLIndent sets the encoder to generate XML in which each element
// begins on a new indented line that starts with prefix and is followed by
// one or more copies of indent according to the nesting depth.
func SetXMLIndent(prefix, indent string) {
std.SetXMLIndent(prefix, indent)
}

21
backend/vendor/github.com/json-iterator/go/Gopkg.lock generated vendored
View File

@@ -1,21 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/modern-go/concurrent"
packages = ["."]
revision = "e0a39a4cb4216ea8db28e22a69f4ec25610d513a"
version = "1.0.0"
[[projects]]
name = "github.com/modern-go/reflect2"
packages = ["."]
revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd"
version = "1.0.1"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "ea54a775e5a354cb015502d2e7aa4b74230fc77e894f34a838b268c25ec8eeb8"
solver-name = "gps-cdcl"
solver-version = 1

15
backend/vendor/github.com/modern-go/reflect2/Gopkg.lock generated vendored
View File

@@ -1,15 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/modern-go/concurrent"
packages = ["."]
revision = "e0a39a4cb4216ea8db28e22a69f4ec25610d513a"
version = "1.0.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "daee8a88b3498b61c5640056665b8b9eea062006f5e596bbb6a3ed9119a11ec7"
solver-name = "gps-cdcl"
solver-version = 1

2
backend/vendor/modules.txt vendored
View File

@@ -40,6 +40,8 @@ github.com/hashicorp/hcl/hcl/token
github.com/hashicorp/hcl/json/parser
github.com/hashicorp/hcl/json/scanner
github.com/hashicorp/hcl/json/token
# github.com/imroc/req v0.2.4
github.com/imroc/req
# github.com/json-iterator/go v1.1.6
github.com/json-iterator/go
# github.com/jtolds/gls v4.20.0+incompatible

3
docker-compose.yml
View File

@@ -8,12 +8,15 @@ services:
CRAWLAB_SERVER_MASTER: "Y" # whether to be master node 是否为主节点，主节点为 Y，工作节点为 N
CRAWLAB_MONGO_HOST: "mongo" # MongoDB host address MongoDB 的地址，在 docker compose 网络中，直接引用服务名称
CRAWLAB_REDIS_ADDRESS: "redis" # Redis host address Redis 的地址，在 docker compose 网络中，直接引用服务名称
# CRAWLAB_SERVER_LANG_NODE: "Y" # 预安装 Node.js 语言环境
ports:
- "8080:8080" # frontend port mapping 前端端口映射
- "8000:8000" # backend port mapping 后端端口映射
depends_on:
- mongo
- redis
volumes:
- "/Users/marvzhang/projects/crawlab-team/crawlab/docker_init.sh:/app/docker_init.sh"
worker:
image: tikazyq/crawlab:latest
container_name: worker

9
docker_init.sh
View File

@@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
# replace default api path to new one
if [ "${CRAWLAB_API_ADDRESS}" = "" ];
@@ -22,5 +22,12 @@ fi
# start nginx
service nginx start
# install languages: Node.js
if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ];
then
echo "installing node.js"
/bin/sh /app/backend/scripts/install-nodejs.sh
fi
# start backend
crawlab

8
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -51,6 +51,10 @@
</el-form>
</el-row>
<el-row class="button-container" v-if="!isView">
<el-button size="normal" v-if="isShowRun" type="danger" @click="onCrawl"
icon="el-icon-video-play" style="margin-right: 10px">
{{$t('Run')}}
</el-button>
<el-upload
v-if="spiderForm.type === 'customized'"
:action="$request.baseUrl + `/spiders/${spiderForm._id}/upload`"
@@ -65,10 +69,6 @@
{{$t('Upload')}}
</el-button>
</el-upload>
<el-button size="normal" v-if="isShowRun" type="danger" @click="onCrawl"
icon="el-icon-video-play">
{{$t('Run')}}
</el-button>
<el-button size="normal" type="success" @click="onSave"
icon="el-icon-check">
{{$t('Save')}}

304
frontend/src/components/Node/NodeInstallation.vue Normal file
View File

@@ -0,0 +1,304 @@
<template>
<div class="node-installation">
<el-form inline>
<el-form-item>
<el-autocomplete
v-if="activeLang.executable_name === 'python'"
v-model="depName"
style="width: 240px"
:placeholder="$t('Search Dependencies')"
:fetchSuggestions="fetchAllDepList"
:minlength="2"
:disabled="isShowInstalled"
@select="onSearch"
/>
<el-input
v-else
v-model="depName"
style="width: 240px"
:placeholder="$t('Search Dependencies')"
:disabled="isShowInstalled"
/>
</el-form-item>
<el-form-item>
<el-button
icon="el-icon-search"
type="success"
:disabled="isShowInstalled"
@click="onSearch"
>
{{$t('Search')}}
</el-button>
</el-form-item>
<el-form-item>
<el-checkbox v-model="isShowInstalled" :label="$t('Show installed')" @change="onIsShowInstalledChange"/>
</el-form-item>
</el-form>
<el-tabs v-model="activeTab" @tab-click="onTabChange">
<el-tab-pane v-for="lang in langList" :key="lang.name" :label="lang.name" :name="lang.executable_name"/>
</el-tabs>
<template v-if="activeLang.installed">
<el-table
height="calc(100vh - 280px)"
:data="computedDepList"
:empty-text="depName ? $t('No Data') : $t('Please search dependencies')"
v-loading="loading"
border
>
<el-table-column
:label="$t('Name')"
prop="name"
width="180"
/>
<el-table-column
:label="!isShowInstalled ? $t('Latest Version') : $t('Version')"
prop="version"
width="100"
/>
<el-table-column
v-if="!isShowInstalled"
:label="$t('Description')"
prop="description"
/>
<el-table-column
:label="$t('Action')"
>
<template slot-scope="scope">
<el-button
v-if="!scope.row.installed"
v-loading="getDepLoading(scope.row)"
:disabled="getDepLoading(scope.row)"
size="mini"
type="primary"
@click="onClickInstallDep(scope.row)"
>
{{$t('Install')}}
</el-button>
<el-button
v-else
v-loading="getDepLoading(scope.row)"
:disabled="getDepLoading(scope.row)"
size="mini"
type="danger"
@click="onClickUninstallDep(scope.row)"
>
{{$t('Uninstall')}}
</el-button>
</template>
</el-table-column>
</el-table>
</template>
<template v-else>
<div class="install-wrapper">
<h3>{{activeLang.name + $t(' is not installed, do you want to install it?')}}</h3>
<el-button
v-loading="isLoadingInstallLang"
:disabled="isLoadingInstallLang"
type="primary"
style="width: 240px;font-weight: bolder;font-size: 18px"
@click="onClickInstallLang"
>
{{$t('Install')}}
</el-button>
</div>
</template>
</div>
</template>
<script>
import {
mapState
} from 'vuex'
export default {
name: 'NodeInstallation',
data () {
return {
activeTab: '',
langList: [],
depName: '',
depList: [],
loading: false,
isShowInstalled: false,
installedDepList: [],
depLoadingDict: {},
isLoadingInstallLang: false
}
},
computed: {
...mapState('node', [
'nodeForm'
]),
activeLang () {
for (let i = 0; i < this.langList.length; i++) {
if (this.langList[i].executable_name === this.activeTab) {
return this.langList[i]
}
}
return {}
},
activeLangName () {
return this.activeLang.executable_name
},
computedDepList () {
if (this.isShowInstalled) {
return this.installedDepList
} else {
return this.depList
}
}
},
methods: {
async getDepList () {
this.loading = true
const res = await this.$request.get(`/nodes/${this.nodeForm._id}/deps`, {
lang: this.activeLang.executable_name,
dep_name: this.depName
})
this.loading = false
this.depList = res.data.data
if (this.activeLangName === 'python') {
// 排序
this.depList = this.depList.sort((a, b) => a.name > b.name ? 1 : -1)
// 异步获取python附加信息
this.depList.map(async dep => {
const res = await this.$request.get(`/system/deps/${this.activeLang.executable_name}/${dep.name}/json`)
dep.version = res.data.data.version
dep.description = res.data.data.description
})
}
},
async getInstalledDepList () {
this.loading = true
const res = await this.$request.get(`/nodes/${this.nodeForm._id}/deps/installed`, {
lang: this.activeLang.executable_name
})
this.loading = false
this.installedDepList = res.data.data
},
async fetchAllDepList (queryString, callback) {
const res = await this.$request.get(`/system/deps/${this.activeLang.executable_name}`, {
dep_name: queryString
})
callback(res.data.data ? res.data.data.map(d => {
return { value: d, label: d }
}) : [])
},
onSearch () {
if (!this.isShowInstalled) {
this.getDepList()
} else {
this.getInstalledDepList()
}
},
onIsShowInstalledChange (val) {
if (val) {
this.getInstalledDepList()
} else {
this.depName = ''
this.depList = []
}
},
async onClickInstallDep (dep) {
const name = dep.name
this.$set(this.depLoadingDict, name, true)
const arr = this.$route.path.split('/')
const id = arr[arr.length - 1]
const data = await this.$request.post(`/nodes/${id}/deps/install`, {
lang: this.activeLang.executable_name,
dep_name: name
})
if (!data || data.error) {
this.$notify.error({
title: this.$t('Installing dependency failed'),
message: this.$t('The dependency installation is unsuccessful: ') + name
})
} else {
this.$notify.success({
title: this.$t('Installing dependency successful'),
message: this.$t('You have successfully installed a dependency: ') + name
})
dep.installed = true
}
this.$set(this.depLoadingDict, name, false)
},
async onClickUninstallDep (dep) {
const name = dep.name
this.$set(this.depLoadingDict, name, true)
const arr = this.$route.path.split('/')
const id = arr[arr.length - 1]
const data = await this.$request.post(`/nodes/${id}/deps/uninstall`, {
lang: this.activeLang.executable_name,
dep_name: name
})
if (!data || data.error) {
this.$notify.error({
title: this.$t('Uninstalling dependency failed'),
message: this.$t('The dependency uninstallation is unsuccessful: ') + name
})
} else {
this.$notify.success({
title: this.$t('Uninstalling dependency successful'),
message: this.$t('You have successfully uninstalled a dependency: ') + name
})
dep.installed = false
}
this.$set(this.depLoadingDict, name, false)
},
getDepLoading (dep) {
const name = dep.name
if (this.depLoadingDict[name] === undefined) {
return false
}
return this.depLoadingDict[name]
},
async onClickInstallLang () {
this.isLoadingInstallLang = true
const res = await this.$request.post(`/nodes/${this.nodeForm._id}/langs/install`, {
lang: this.activeLang.executable_name
})
if (!res || res.error) {
this.$notify.error({
title: this.$t('Installing language failed'),
message: this.$t('The language installation is unsuccessful: ') + this.activeLang.name
})
} else {
this.$notify.success({
title: this.$t('Installing language successful'),
message: this.$t('You have successfully installed a language: ') + this.activeLang.name
})
}
this.isLoadingInstallLang = false
},
onTabChange () {
if (this.isShowInstalled) {
this.getInstalledDepList()
} else {
this.depName = ''
this.depList = []
}
}
},
async created () {
const arr = this.$route.path.split('/')
const id = arr[arr.length - 1]
const res = await this.$request.get(`/nodes/${id}/langs`)
this.langList = res.data.data
this.activeTab = this.langList[0].executable_name || ''
}
}
</script>
<style scoped>
.node-installation >>> .el-button .el-loading-spinner {
margin-top: -13px;
height: 28px;
}
.node-installation >>> .el-button .el-loading-spinner .circular {
width: 28px;
height: 28px;
}
</style>

22
frontend/src/i18n/zh.js
View File

@@ -65,6 +65,8 @@ export default {
'Back': '返回',
'New File': '新建文件',
'Rename': '重命名',
'Install': '安装',
'Uninstall': '卸载',
// 主页
'Total Tasks': '总任务数',
@@ -85,6 +87,8 @@ export default {
'Node Network': '节点拓扑图',
'Master': '主节点',
'Worker': '工作节点',
'Installation': '安装',
'Search Dependencies': '搜索依赖',
// 节点列表
'IP': 'IP地址',
@@ -262,6 +266,8 @@ export default {
'ARCH': '操作架构',
'Number of CPU': 'CPU数',
'Executables': '执行文件',
'Latest Version': '最新版本',
'Version': '版本',
// 弹出框
'Notification': '提示',
@@ -290,7 +296,23 @@ export default {
'NOTE: When uploading a zip file, please zip your spider files from the ROOT DIRECTORY.': '注意: 上传 zip 文件时，请从 根目录 下开始压缩爬虫文件。',
'English': 'English',
'Are you sure to delete the schedule task?': '确定删除定时任务?',
' is not installed, do you want to install it?': ' 还没有安装，您是否打算安装它?',
'Disclaimer': '免责声明',
'Please search dependencies': '请搜索依赖',
'No Data': '暂无数据',
'Show installed': '查看已安装',
'Installing dependency successful': '安装依赖成功',
'Installing dependency failed': '安装依赖失败',
'You have successfully installed a dependency: ': '您已成功安装依赖: ',
'The dependency installation is unsuccessful: ': '安装依赖失败: ',
'Uninstalling dependency successful': '卸载依赖成功',
'Uninstalling dependency failed': '卸载依赖失败',
'You have successfully uninstalled a dependency: ': '您已成功卸载依赖: ',
'The dependency uninstallation is unsuccessful: ': '卸载依赖失败: ',
'Installing language successful': '安装语言成功',
'Installing language failed': '安装语言失败',
'You have successfully installed a language: ': '您已成功安装语言: ',
'The language installation is unsuccessful: ': '安装语言失败: ',
// 登录
'Sign in': '登录',

2
frontend/src/store/modules/node.js
View File

@@ -3,7 +3,7 @@ import request from '../../api/request'
const state = {
// NodeList
nodeList: [],
nodeForm: { _id: {} },
nodeForm: {},
// spider to deploy/run
activeSpider: {}

11
frontend/src/views/node/NodeDetail.vue
View File

@@ -13,6 +13,9 @@
<el-tab-pane :label="$t('Overview')" name="overview">
<node-overview></node-overview>
</el-tab-pane>
<el-tab-pane :label="$t('Installation')" name="installation">
<node-installation></node-installation>
</el-tab-pane>
<el-tab-pane :label="$t('Deployed Spiders')" name="spiders" v-if="false">
{{$t('Deployed Spiders')}}
</el-tab-pane>
@@ -25,11 +28,13 @@ import {
mapState
} from 'vuex'
import NodeOverview from '../../components/Overview/NodeOverview'
import NodeInstallation from '../../components/Node/NodeInstallation'
export default {
name: 'NodeDetail',
components: {
NodeOverview
NodeOverview,
NodeInstallation
},
data () {
return {
@@ -43,7 +48,9 @@ export default {
])
},
methods: {
onTabClick () {
onTabClick (name) {
if (name === 'installation') {
}
},
onNodeChange (id) {
this.$router.push(`/nodes/${id}`)

1
jenkins/master/docker-compose.yaml
View File

@@ -9,6 +9,7 @@ services:
CRAWLAB_MONGO_HOST: "mongo"
CRAWLAB_REDIS_ADDRESS: "redis"
CRAWLAB_LOG_PATH: "/var/logs/crawlab"
CRAWLAB_SETTING_ALLOWREGISTER: "Y"
ports:
- "8080:8080" # frontend
- "8000:8000" # backend

178
wait-for-it.sh
View File

@@ -1,178 +0,0 @@
#!/usr/bin/env bash
# Use this script to test if a given TCP host/port are available
WAITFORIT_cmdname=${0##*/}
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
usage()
{
cat << USAGE >&2
Usage:
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-h HOST | --host=HOST Host or IP under test
-p PORT | --port=PORT TCP port under test
Alternatively, you specify the host and port as host:port
-s | --strict Only execute subcommand if the test succeeds
-q | --quiet Don't output any status messages
-t TIMEOUT | --timeout=TIMEOUT
Timeout in seconds, zero for no timeout
-- COMMAND ARGS Execute command with args after the test finishes
USAGE
exit 1
}
wait_for()
{
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
else
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
fi
WAITFORIT_start_ts=$(date +%s)
while :
do
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
WAITFORIT_result=$?
else
(echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
WAITFORIT_result=$?
fi
if [[ $WAITFORIT_result -eq 0 ]]; then
WAITFORIT_end_ts=$(date +%s)
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
break
fi
sleep 1
done
return $WAITFORIT_result
}
wait_for_wrapper()
{
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
else
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
fi
WAITFORIT_PID=$!
trap "kill -INT -$WAITFORIT_PID" INT
wait $WAITFORIT_PID
WAITFORIT_RESULT=$?
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
fi
return $WAITFORIT_RESULT
}
# process arguments
while [[ $# -gt 0 ]]
do
case "$1" in
*:* )
WAITFORIT_hostport=(${1//:/ })
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
shift 1
;;
--child)
WAITFORIT_CHILD=1
shift 1
;;
-q | --quiet)
WAITFORIT_QUIET=1
shift 1
;;
-s | --strict)
WAITFORIT_STRICT=1
shift 1
;;
-h)
WAITFORIT_HOST="$2"
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
shift 2
;;
--host=*)
WAITFORIT_HOST="${1#*=}"
shift 1
;;
-p)
WAITFORIT_PORT="$2"
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
shift 2
;;
--port=*)
WAITFORIT_PORT="${1#*=}"
shift 1
;;
-t)
WAITFORIT_TIMEOUT="$2"
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
shift 2
;;
--timeout=*)
WAITFORIT_TIMEOUT="${1#*=}"
shift 1
;;
--)
shift
WAITFORIT_CLI=("$@")
break
;;
--help)
usage
;;
*)
echoerr "Unknown argument: $1"
usage
;;
esac
done
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
echoerr "Error: you need to provide a host and port to test."
usage
fi
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
# check to see if timeout is from busybox?
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
WAITFORIT_ISBUSY=1
WAITFORIT_BUSYTIMEFLAG="-t"
else
WAITFORIT_ISBUSY=0
WAITFORIT_BUSYTIMEFLAG=""
fi
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
wait_for
WAITFORIT_RESULT=$?
exit $WAITFORIT_RESULT
else
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
wait_for_wrapper
WAITFORIT_RESULT=$?
else
wait_for
WAITFORIT_RESULT=$?
fi
fi
if [[ $WAITFORIT_CLI != "" ]]; then
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
exit $WAITFORIT_RESULT
fi
exec "${WAITFORIT_CLI[@]}"
else
exit $WAITFORIT_RESULT
fi