diff --git a/CHANGELOG.md b/CHANGELOG.md index aa2682ce..1275b9fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# 0.4.3 (unknown) + +### Features / Enhancement +- **Dependency Installation**. Allow users to install/uninstall dependencies and add programming languages (Node.js only for now) on the platform web interface. +- **Pre-install Programming Languages in Docker**. Allow Docker users to set `CRAWLAB_SERVER_LANG_NODE` as `Y` to pre-install `Node.js` environments. + + # 0.4.2 (2019-12-26) ### Features / Enhancement - **Disclaimer**. Added page for Disclaimer. diff --git a/Dockerfile b/Dockerfile index cbc5c65d..73883c64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,9 @@ ADD . /app # set as non-interactive ENV DEBIAN_FRONTEND noninteractive +# set CRAWLAB_IS_DOCKER +ENV CRAWLAB_IS_DOCKER Y + # install packages RUN apt-get update \ && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip \ @@ -56,4 +59,4 @@ EXPOSE 8080 EXPOSE 8000 # start backend -CMD ["/bin/sh", "/app/docker_init.sh"] +CMD ["/bin/bash", "/app/docker_init.sh"] diff --git a/Dockerfile.local b/Dockerfile.local index d99010a4..59b8736d 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -57,4 +57,4 @@ EXPOSE 8080 EXPOSE 8000 # start backend -CMD ["/bin/sh", "/app/docker_init.sh"] +CMD ["/bin/bash", "/app/docker_init.sh"] diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 5ada78f6..a51a34c9 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -26,12 +26,15 @@ server: # mac地址 或者 ip地址，如果是ip，则需要手动指定IP type: "mac" ip: "" + lang: # 安装语言环境, Y 为安装，N 为不安装，只对 Docker 有效 + python: "Y" + node: "N" spider: path: "/app/spiders" task: workers: 4 other: tmppath: "/tmp" -version: 0.4.2 +version: 0.4.3 setting: allowRegister: "N" \ No newline at end of file diff --git a/backend/constants/rpc.go b/backend/constants/rpc.go new file mode 100644 index 00000000..6eebf0d5 --- /dev/null +++ b/backend/constants/rpc.go @@ -0,0 +1,9 @@ +package constants + +const ( + RpcInstallLang = "install_lang" + RpcInstallDep = "install_dep" + RpcUninstallDep = "uninstall_dep" + RpcGetDepList = "get_dep_list" + RpcGetInstalledDepList = "get_installed_dep_list" +) diff --git a/backend/constants/system.go b/backend/constants/system.go index 59c39787..bec8b8c5 100644 --- a/backend/constants/system.go +++ b/backend/constants/system.go @@ -5,3 +5,9 @@ const ( Linux = "linux" Darwin = "darwin" ) + +const ( + Python = "python" + Nodejs = "node" + Java = "java" +) diff --git a/backend/database/redis.go b/backend/database/redis.go index b165aaa3..4ecabbbd 100644 --- a/backend/database/redis.go +++ b/backend/database/redis.go @@ -36,6 +36,19 @@ func (r *Redis) RPush(collection string, value interface{}) error { defer utils.Close(c) if _, err := c.Do("RPUSH", collection, value); err != nil { + log.Error(err.Error()) + debug.PrintStack() + return err + } + return nil +} + +func (r *Redis) LPush(collection string, value interface{}) error { + c := r.pool.Get() + defer utils.Close(c) + + if _, err := c.Do("RPUSH", collection, value); err != nil { + log.Error(err.Error()) debug.PrintStack() return err } @@ -58,6 +71,7 @@ func (r *Redis) HSet(collection string, key string, value string) error { defer utils.Close(c) if _, err := c.Do("HSET", collection, key, value); err != nil { + log.Error(err.Error()) debug.PrintStack() return err } @@ -70,6 +84,8 @@ func (r *Redis) HGet(collection string, key string) (string, error) { value, err2 := redis.String(c.Do("HGET", collection, key)) if err2 != nil { + log.Error(err2.Error()) + debug.PrintStack() return value, err2 } return value, nil @@ -80,6 +96,8 @@ func (r *Redis) HDel(collection string, key string) error { defer utils.Close(c) if _, err := c.Do("HDEL", collection, key); err != nil { + log.Error(err.Error()) + debug.PrintStack() return err } return nil @@ -91,11 +109,29 @@ func (r *Redis) HKeys(collection string) ([]string, error) { value, err2 := redis.Strings(c.Do("HKeys", collection)) if err2 != nil { + log.Error(err2.Error()) + debug.PrintStack() return []string{}, err2 } return value, nil } +func (r *Redis) BRPop(collection string, timeout int) (string, error) { + if timeout <= 0 { + timeout = 60 + } + c := r.pool.Get() + defer utils.Close(c) + + values, err := redis.Strings(c.Do("BRPOP", collection, timeout)) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return "", err + } + return values[1], nil +} + func NewRedisPool() *redis.Pool { var address = viper.GetString("redis.address") var port = viper.GetString("redis.port") @@ -112,8 +148,8 @@ func NewRedisPool() *redis.Pool { Dial: func() (conn redis.Conn, e error) { return redis.DialURL(url, redis.DialConnectTimeout(time.Second*10), - redis.DialReadTimeout(time.Second*10), - redis.DialWriteTimeout(time.Second*15), + redis.DialReadTimeout(time.Second*600), + redis.DialWriteTimeout(time.Second*10), ) }, TestOnBorrow: func(c redis.Conn, t time.Time) error { diff --git a/backend/entity/system.go b/backend/entity/system.go index dff637b7..ac3e9dec 100644 --- a/backend/entity/system.go +++ b/backend/entity/system.go @@ -13,3 +13,18 @@ type Executable struct { FileName string `json:"file_name"` DisplayName string `json:"display_name"` } + +type Lang struct { + Name string `json:"name"` + ExecutableName string `json:"executable_name"` + ExecutablePath string `json:"executable_path"` + DepExecutablePath string `json:"dep_executable_path"` + Installed bool `json:"installed"` +} + +type Dependency struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Installed bool `json:"installed"` +} diff --git a/backend/go.mod b/backend/go.mod index d59b6d41..89bbdbbc 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -11,6 +11,7 @@ require ( github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/gomodule/redigo v2.0.0+incompatible + github.com/imroc/req v0.2.4 github.com/leodido/go-urn v1.1.0 // indirect github.com/pkg/errors v0.8.1 github.com/satori/go.uuid v1.2.0 diff --git a/backend/go.sum b/backend/go.sum index 55a56852..e4386cc9 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -66,6 +66,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imroc/req v0.2.4 h1:8XbvaQpERLAJV6as/cB186DtH5f0m5zAOtHEaTQ4ac0= +github.com/imroc/req v0.2.4/go.mod h1:J9FsaNHDTIVyW/b5r6/Df5qKEEEq2WzZKIgKSajd1AE= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= diff --git a/backend/main.go b/backend/main.go index 955fb77c..7dd5046e 100644 --- a/backend/main.go +++ b/backend/main.go @@ -31,24 +31,24 @@ func main() { log.Error("init config error:" + err.Error()) panic(err) } - log.Info("初始化配置成功") + log.Info("initialized config successfully") // 初始化日志设置 logLevel := viper.GetString("log.level") if logLevel != "" { log.SetLevelFromString(logLevel) } - log.Info("初始化日志设置成功") + log.Info("initialized log config successfully") if viper.GetString("log.isDeletePeriodically") == "Y" { err := services.InitDeleteLogPeriodically() if err != nil { - log.Error("Init DeletePeriodically Failed") + log.Error("init DeletePeriodically failed") panic(err) } - log.Info("初始化定期清理日志配置成功") + log.Info("initialized periodically cleaning log successfully") } else { - log.Info("默认未开启定期清理日志配置") + log.Info("periodically cleaning log is switched off") } // 初始化Mongodb数据库 @@ -57,7 +57,7 @@ func main() { debug.PrintStack() panic(err) } - log.Info("初始化Mongodb数据库成功") + log.Info("initialized MongoDB successfully") // 初始化Redis数据库 if err := database.InitRedis(); err != nil { @@ -65,7 +65,7 @@ func main() { debug.PrintStack() panic(err) } - log.Info("初始化Redis数据库成功") + log.Info("initialized Redis successfully") if model.IsMaster() { // 初始化定时任务 @@ -74,8 +74,8 @@ func main() { debug.PrintStack() panic(err) } - log.Info("初始化定时任务成功") } + log.Info("initialized schedule successfully") // 初始化任务执行器 if err := services.InitTaskExecutor(); err != nil { @@ -83,14 +83,14 @@ func main() { debug.PrintStack() panic(err) } - log.Info("初始化任务执行器成功") + log.Info("initialized task executor successfully") // 初始化节点服务 if err := services.InitNodeService(); err != nil { log.Error("init node service error:" + err.Error()) panic(err) } - log.Info("初始化节点配置成功") + log.Info("initialized node service successfully") // 初始化爬虫服务 if err := services.InitSpiderService(); err != nil { @@ -98,7 +98,7 @@ func main() { debug.PrintStack() panic(err) } - log.Info("初始化爬虫服务成功") + log.Info("initialized spider service successfully") // 初始化用户服务 if err := services.InitUserService(); err != nil { @@ -106,7 +106,23 @@ func main() { debug.PrintStack() panic(err) } - log.Info("初始化用户服务成功") + log.Info("initialized user service successfully") + + // 初始化依赖服务 + if err := services.InitDepsFetcher(); err != nil { + log.Error("init dependency fetcher error:" + err.Error()) + debug.PrintStack() + panic(err) + } + log.Info("initialized dependency fetcher successfully") + + // 初始化RPC服务 + if err := services.InitRpcService(); err != nil { + log.Error("init rpc service error:" + err.Error()) + debug.PrintStack() + panic(err) + } + log.Info("initialized rpc service successfully") // 以下为主节点服务 if model.IsMaster() { @@ -122,12 +138,18 @@ func main() { { // 路由 // 节点 - authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 - authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 - authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 - authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 - authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 - authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 + authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 + authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 + authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 + authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 + authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 + authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 + authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 + authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 + authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表 + authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖 + authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖 + authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言 // 爬虫 authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 @@ -184,6 +206,9 @@ func main() { authGroup.GET("/me", routes.GetMe) // 获取自己账户 // release版本 authGroup.GET("/version", routes.GetVersion) // 获取发布的版本 + // 系统 + authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表 + authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON } } diff --git a/backend/routes/system.go b/backend/routes/system.go new file mode 100644 index 00000000..b4e130a9 --- /dev/null +++ b/backend/routes/system.go @@ -0,0 +1,315 @@ +package routes + +import ( + "crawlab/constants" + "crawlab/entity" + "crawlab/services" + "fmt" + "github.com/gin-gonic/gin" + "net/http" + "strings" +) + +func GetLangList(c *gin.Context) { + nodeId := c.Param("id") + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: services.GetLangList(nodeId), + }) +} + +func GetDepList(c *gin.Context) { + nodeId := c.Param("id") + lang := c.Query("lang") + depName := c.Query("dep_name") + + var depList []entity.Dependency + if lang == constants.Python { + list, err := services.GetPythonDepList(nodeId, depName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list + } else if lang == constants.Nodejs { + list, err := services.GetNodejsDepList(nodeId, depName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang)) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: depList, + }) +} + +func GetInstalledDepList(c *gin.Context) { + nodeId := c.Param("id") + lang := c.Query("lang") + var depList []entity.Dependency + if lang == constants.Python { + if services.IsMasterNode(nodeId) { + list, err := services.GetPythonLocalInstalledDepList(nodeId) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list + } else { + list, err := services.GetPythonRemoteInstalledDepList(nodeId) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list + } + } else if lang == constants.Nodejs { + if services.IsMasterNode(nodeId) { + list, err := services.GetNodejsLocalInstalledDepList(nodeId) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list + } else { + list, err := services.GetNodejsRemoteInstalledDepList(nodeId) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list + } + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang)) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: depList, + }) +} + +func GetAllDepList(c *gin.Context) { + lang := c.Param("lang") + depName := c.Query("dep_name") + + // 获取所有依赖列表 + var list []string + if lang == constants.Python { + _list, err := services.GetPythonDepListFromRedis() + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + list = _list + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang)) + return + } + + // 过滤依赖列表 + var depList []string + for _, name := range list { + if strings.HasPrefix(strings.ToLower(name), strings.ToLower(depName)) { + depList = append(depList, name) + } + } + + // 只取前20 + var returnList []string + for i, name := range depList { + if i >= 10 { + break + } + returnList = append(returnList, name) + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: returnList, + }) +} + +func InstallDep(c *gin.Context) { + type ReqBody struct { + Lang string `json:"lang"` + DepName string `json:"dep_name"` + } + + nodeId := c.Param("id") + + var reqBody ReqBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + if reqBody.Lang == constants.Python { + if services.IsMasterNode(nodeId) { + _, err := services.InstallPythonLocalDep(reqBody.DepName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } else { + _, err := services.InstallPythonRemoteDep(nodeId, reqBody.DepName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } + } else if reqBody.Lang == constants.Nodejs { + if services.IsMasterNode(nodeId) { + _, err := services.InstallNodejsLocalDep(reqBody.DepName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } else { + _, err := services.InstallNodejsRemoteDep(nodeId, reqBody.DepName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang)) + return + } + + // TODO: check if install is successful + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func UninstallDep(c *gin.Context) { + type ReqBody struct { + Lang string `json:"lang"` + DepName string `json:"dep_name"` + } + + nodeId := c.Param("id") + + var reqBody ReqBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleError(http.StatusBadRequest, c, err) + } + + if reqBody.Lang == constants.Python { + if services.IsMasterNode(nodeId) { + _, err := services.UninstallPythonLocalDep(reqBody.DepName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } else { + _, err := services.UninstallPythonRemoteDep(nodeId, reqBody.DepName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } + } else if reqBody.Lang == constants.Nodejs { + if services.IsMasterNode(nodeId) { + _, err := services.UninstallNodejsLocalDep(reqBody.DepName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } else { + _, err := services.UninstallNodejsRemoteDep(nodeId, reqBody.DepName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang)) + return + } + + // TODO: check if uninstall is successful + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func GetDepJson(c *gin.Context) { + depName := c.Param("dep_name") + lang := c.Param("lang") + + var dep entity.Dependency + if lang == constants.Python { + _dep, err := services.FetchPythonDepInfo(depName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + } + dep = _dep + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang)) + return + } + + c.Header("Cache-Control", "max-age=86400") + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: dep, + }) +} + +func InstallLang(c *gin.Context) { + type ReqBody struct { + Lang string `json:"lang"` + } + + nodeId := c.Param("id") + + var reqBody ReqBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + if reqBody.Lang == constants.Nodejs { + if services.IsMasterNode(nodeId) { + _, err := services.InstallNodejsLocalLang() + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } else { + _, err := services.InstallNodejsRemoteLang(nodeId) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang)) + return + } + + // TODO: check if install is successful + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/scripts/install-nodejs.sh b/backend/scripts/install-nodejs.sh new file mode 100644 index 00000000..1ca73b2d --- /dev/null +++ b/backend/scripts/install-nodejs.sh @@ -0,0 +1,17 @@ +#!/bin/env bash + +# install nvm +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.2/install.sh | bash +export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm + +# install Node.js v8.12 +nvm install 8.12 + +# create soft links +ln -s $HOME/.nvm/versions/node/v8.12.0/bin/npm /usr/local/bin/npm +ln -s $HOME/.nvm/versions/node/v8.12.0/bin/node /usr/local/bin/node + +# environments manipulation +export NODE_PATH=$HOME.nvm/versions/node/v8.12.0/lib/node_modules +export PATH=$NODE_PATH:$PATH \ No newline at end of file diff --git a/backend/services/node.go b/backend/services/node.go index d14ce4ae..515ce9c9 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" "github.com/apex/log" + "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "github.com/gomodule/redigo/redis" "runtime/debug" @@ -116,7 +117,7 @@ func handleNodeInfo(key string, data *Data) { defer s.Close() var node model.Node - if err := c.Find(bson.M{"key": key}).One(&node); err != nil { + if err := c.Find(bson.M{"key": key}).One(&node); err != nil && err == mgo.ErrNotFound { // 数据库不存在该节点 node = model.Node{ Key: key, @@ -133,7 +134,7 @@ func handleNodeInfo(key string, data *Data) { log.Errorf(err.Error()) return } - } else { + } else if node.Key != "" { // 数据库存在该节点 node.Status = constants.StatusOnline node.UpdateTs = time.Now() @@ -168,33 +169,27 @@ func UpdateNodeData() { return } - //先获取所有Redis的nodekey - list, _ := database.RedisClient.HKeys("nodes") - - if i := utils.Contains(list, key); i == false { - // 构造节点数据 - data := Data{ - Key: key, - Mac: mac, - Ip: ip, - Master: model.IsMaster(), - UpdateTs: time.Now(), - UpdateTsUnix: time.Now().Unix(), - } - - // 注册节点到Redis - dataBytes, err := json.Marshal(&data) - if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return - } - if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil { - log.Errorf(err.Error()) - return - } + // 构造节点数据 + data := Data{ + Key: key, + Mac: mac, + Ip: ip, + Master: model.IsMaster(), + UpdateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), } + // 注册节点到Redis + dataBytes, err := json.Marshal(&data) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } + if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil { + log.Errorf(err.Error()) + return + } } func MasterNodeCallback(message redis.Message) (err error) { diff --git a/backend/services/rpc.go b/backend/services/rpc.go new file mode 100644 index 00000000..9b6a5b74 --- /dev/null +++ b/backend/services/rpc.go @@ -0,0 +1,231 @@ +package services + +import ( + "crawlab/constants" + "crawlab/database" + "crawlab/entity" + "crawlab/model" + "crawlab/utils" + "encoding/json" + "fmt" + "github.com/apex/log" + uuid "github.com/satori/go.uuid" + "runtime/debug" +) + +type RpcMessage struct { + Id string `json:"id"` + Method string `json:"method"` + Params map[string]string `json:"params"` + Result string `json:"result"` +} + +func RpcServerInstallLang(msg RpcMessage) RpcMessage { + lang := GetRpcParam("lang", msg.Params) + if lang == constants.Nodejs { + output, _ := InstallNodejsLocalLang() + msg.Result = output + } + return msg +} + +func RpcClientInstallLang(nodeId string, lang string) (output string, err error) { + params := map[string]string{} + params["lang"] = lang + + data, err := RpcClientFunc(nodeId, constants.RpcInstallLang, params, 600)() + if err != nil { + return + } + + output = data + + return +} + +func RpcServerInstallDep(msg RpcMessage) RpcMessage { + lang := GetRpcParam("lang", msg.Params) + depName := GetRpcParam("dep_name", msg.Params) + if lang == constants.Python { + output, _ := InstallPythonLocalDep(depName) + msg.Result = output + } + return msg +} + +func RpcClientInstallDep(nodeId string, lang string, depName string) (output string, err error) { + params := map[string]string{} + params["lang"] = lang + params["dep_name"] = depName + + data, err := RpcClientFunc(nodeId, constants.RpcInstallDep, params, 10)() + if err != nil { + return + } + + output = data + + return +} + +func RpcServerUninstallDep(msg RpcMessage) RpcMessage { + lang := GetRpcParam("lang", msg.Params) + depName := GetRpcParam("dep_name", msg.Params) + if lang == constants.Python { + output, _ := UninstallPythonLocalDep(depName) + msg.Result = output + } + return msg +} + +func RpcClientUninstallDep(nodeId string, lang string, depName string) (output string, err error) { + params := map[string]string{} + params["lang"] = lang + params["dep_name"] = depName + + data, err := RpcClientFunc(nodeId, constants.RpcUninstallDep, params, 60)() + if err != nil { + return + } + + output = data + + return +} + +func RpcServerGetInstalledDepList(nodeId string, msg RpcMessage) RpcMessage { + lang := GetRpcParam("lang", msg.Params) + if lang == constants.Python { + depList, _ := GetPythonLocalInstalledDepList(nodeId) + resultStr, _ := json.Marshal(depList) + msg.Result = string(resultStr) + } else if lang == constants.Nodejs { + depList, _ := GetNodejsLocalInstalledDepList(nodeId) + resultStr, _ := json.Marshal(depList) + msg.Result = string(resultStr) + } + return msg +} + +func RpcClientGetInstalledDepList(nodeId string, lang string) (list []entity.Dependency, err error) { + params := map[string]string{} + params["lang"] = lang + + data, err := RpcClientFunc(nodeId, constants.RpcGetInstalledDepList, params, 10)() + if err != nil { + return + } + + // 反序列化结果 + if err := json.Unmarshal([]byte(data), &list); err != nil { + return list, err + } + + return +} + +func RpcClientFunc(nodeId string, method string, params map[string]string, timeout int) func() (string, error) { + return func() (result string, err error) { + // 请求ID + id := uuid.NewV4().String() + + // 构造RPC消息 + msg := RpcMessage{ + Id: id, + Method: method, + Params: params, + Result: "", + } + + // 发送RPC消息 + msgStr := ObjectToString(msg) + if err := database.RedisClient.LPush(fmt.Sprintf("rpc:%s", nodeId), msgStr); err != nil { + return result, err + } + + // 获取RPC回复消息 + dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", nodeId), timeout) + if err != nil { + return result, err + } + + // 反序列化消息 + if err := json.Unmarshal([]byte(dataStr), &msg); err != nil { + return result, err + } + + return msg.Result, err + } +} + +func GetRpcParam(key string, params map[string]string) string { + return params[key] +} + +func ObjectToString(params interface{}) string { + bytes, _ := json.Marshal(params) + return utils.BytesToString(bytes) +} + +var IsRpcStopped = false + +func StopRpcService() { + IsRpcStopped = true +} + +func InitRpcService() error { + go func() { + for { + // 获取当前节点 + node, err := model.GetCurrentNode() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + + // 获取获取消息队列信息 + dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 300) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + + // 反序列化消息 + var msg RpcMessage + if err := json.Unmarshal([]byte(dataStr), &msg); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + + // 根据Method调用本地方法 + var replyMsg RpcMessage + if msg.Method == constants.RpcInstallDep { + replyMsg = RpcServerInstallDep(msg) + } else if msg.Method == constants.RpcUninstallDep { + replyMsg = RpcServerUninstallDep(msg) + } else if msg.Method == constants.RpcInstallLang { + replyMsg = RpcServerInstallLang(msg) + } else if msg.Method == constants.RpcGetInstalledDepList { + replyMsg = RpcServerGetInstalledDepList(node.Id.Hex(), msg) + } else { + continue + } + + // 发送返回消息 + if err := database.RedisClient.LPush(fmt.Sprintf("rpc:%s", node.Id.Hex()), ObjectToString(replyMsg)); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + + // 如果停止RPC服务，则返回 + if IsRpcStopped { + return + } + } + }() + return nil +} diff --git a/backend/services/system.go b/backend/services/system.go index 92f9cf96..12b8744c 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -4,28 +4,42 @@ import ( "crawlab/constants" "crawlab/database" "crawlab/entity" + "crawlab/lib/cron" "crawlab/model" "crawlab/utils" "encoding/json" + "errors" + "fmt" + "github.com/apex/log" + "github.com/imroc/req" + "os/exec" + "path" + "regexp" + "runtime/debug" + "sort" + "strings" + "sync" ) +// 系统信息 chan 映射 var SystemInfoChanMap = utils.NewChanMap() -func GetRemoteSystemInfo(id string) (sysInfo entity.SystemInfo, err error) { +// 从远端获取系统信息 +func GetRemoteSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) { // 发送消息 msg := entity.NodeMessage{ Type: constants.MsgTypeGetSystemInfo, - NodeId: id, + NodeId: nodeId, } // 序列化 msgBytes, _ := json.Marshal(&msg) - if _, err := database.RedisClient.Publish("nodes:"+id, utils.BytesToString(msgBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:"+nodeId, utils.BytesToString(msgBytes)); err != nil { return entity.SystemInfo{}, err } // 通道 - ch := SystemInfoChanMap.ChanBlocked(id) + ch := SystemInfoChanMap.ChanBlocked(nodeId) // 等待响应，阻塞 sysInfoStr := <-ch @@ -38,11 +52,531 @@ func GetRemoteSystemInfo(id string) (sysInfo entity.SystemInfo, err error) { return sysInfo, nil } -func GetSystemInfo(id string) (sysInfo entity.SystemInfo, err error) { - if IsMasterNode(id) { +// 获取系统信息 +func GetSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) { + if IsMasterNode(nodeId) { sysInfo, err = model.GetLocalSystemInfo() } else { - sysInfo, err = GetRemoteSystemInfo(id) + sysInfo, err = GetRemoteSystemInfo(nodeId) } return } + +// 获取语言列表 +func GetLangList(nodeId string) []entity.Lang { + list := []entity.Lang{ + {Name: "Python", ExecutableName: "python", ExecutablePath: "/usr/local/bin/python", DepExecutablePath: "/usr/local/bin/pip"}, + {Name: "Node.js", ExecutableName: "node", ExecutablePath: "/usr/local/bin/node", DepExecutablePath: "/usr/local/bin/npm"}, + //{Name: "Java", ExecutableName: "java", ExecutablePath: "/usr/local/bin/java"}, + } + for i, lang := range list { + list[i].Installed = IsInstalledLang(nodeId, lang) + } + return list +} + +// 根据语言名获取语言实例 +func GetLangFromLangName(nodeId string, name string) entity.Lang { + langList := GetLangList(nodeId) + for _, lang := range langList { + if lang.ExecutableName == name { + return lang + } + } + return entity.Lang{} +} + +// 是否已安装该依赖 +func IsInstalledLang(nodeId string, lang entity.Lang) bool { + sysInfo, err := GetSystemInfo(nodeId) + if err != nil { + return false + } + for _, exec := range sysInfo.Executables { + if exec.Path == lang.ExecutablePath { + return true + } + } + return false +} + +// 是否已安装该依赖 +func IsInstalledDep(installedDepList []entity.Dependency, dep entity.Dependency) bool { + for _, _dep := range installedDepList { + if strings.ToLower(_dep.Name) == strings.ToLower(dep.Name) { + return true + } + } + return false +} + +// 初始化函数 +func InitDepsFetcher() error { + c := cron.New(cron.WithSeconds()) + c.Start() + if _, err := c.AddFunc("0 */5 * * * *", UpdatePythonDepList); err != nil { + return err + } + + go func() { + UpdatePythonDepList() + }() + return nil +} + +// ========= +// Python +// ========= + +type PythonDepJsonData struct { + Info PythonDepJsonDataInfo `json:"info"` +} + +type PythonDepJsonDataInfo struct { + Name string `json:"name"` + Summary string `json:"summary"` + Version string `json:"version"` +} + +type PythonDepNameDict struct { + Name string `json:"name"` + Weight int `json:"weight"` +} + +type PythonDepNameDictSlice []PythonDepNameDict + +func (s PythonDepNameDictSlice) Len() int { return len(s) } +func (s PythonDepNameDictSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s PythonDepNameDictSlice) Less(i, j int) bool { return s[i].Weight > s[j].Weight } + +// 获取Python本地依赖列表 +func GetPythonDepList(nodeId string, searchDepName string) ([]entity.Dependency, error) { + var list []entity.Dependency + + // 先从 Redis 获取 + depList, err := GetPythonDepListFromRedis() + if err != nil { + return list, err + } + + // 过滤相似的依赖 + var depNameList PythonDepNameDictSlice + for _, depName := range depList { + if strings.HasPrefix(strings.ToLower(depName), strings.ToLower(searchDepName)) { + var weight int + if strings.ToLower(depName) == strings.ToLower(searchDepName) { + weight = 3 + } else if strings.HasPrefix(strings.ToLower(depName), strings.ToLower(searchDepName)) { + weight = 2 + } else { + weight = 1 + } + depNameList = append(depNameList, PythonDepNameDict{ + Name: depName, + Weight: weight, + }) + } + } + + // 获取已安装依赖列表 + var installedDepList []entity.Dependency + if IsMasterNode(nodeId) { + installedDepList, err = GetPythonLocalInstalledDepList(nodeId) + if err != nil { + return list, err + } + } else { + installedDepList, err = GetPythonRemoteInstalledDepList(nodeId) + if err != nil { + return list, err + } + } + + // 根据依赖名排序 + sort.Stable(depNameList) + + // 遍历依赖名列表，取前20个 + for i, depNameDict := range depNameList { + if i > 20 { + break + } + dep := entity.Dependency{ + Name: depNameDict.Name, + } + dep.Installed = IsInstalledDep(installedDepList, dep) + list = append(list, dep) + } + + // 从依赖源获取信息 + //list, err = GetPythonDepListWithInfo(list) + + return list, nil +} + +// 获取Python依赖的源数据信息 +func GetPythonDepListWithInfo(depList []entity.Dependency) ([]entity.Dependency, error) { + var goSync sync.WaitGroup + for i, dep := range depList { + if i > 10 { + break + } + goSync.Add(1) + go func(i int, dep entity.Dependency, depList []entity.Dependency, n *sync.WaitGroup) { + url := fmt.Sprintf("https://pypi.org/pypi/%s/json", dep.Name) + res, err := req.Get(url) + if err != nil { + n.Done() + return + } + var data PythonDepJsonData + if err := res.ToJSON(&data); err != nil { + n.Done() + return + } + depList[i].Version = data.Info.Version + depList[i].Description = data.Info.Summary + n.Done() + }(i, dep, depList, &goSync) + } + goSync.Wait() + return depList, nil +} + +func FetchPythonDepInfo(depName string) (entity.Dependency, error) { + url := fmt.Sprintf("https://pypi.org/pypi/%s/json", depName) + res, err := req.Get(url) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return entity.Dependency{}, err + } + var data PythonDepJsonData + if err := res.ToJSON(&data); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return entity.Dependency{}, err + } + dep := entity.Dependency{ + Name: depName, + Version: data.Info.Version, + Description: data.Info.Summary, + } + return dep, nil +} + +// 从Redis获取Python依赖列表 +func GetPythonDepListFromRedis() ([]string, error) { + var list []string + + // 从 Redis 获取字符串 + rawData, err := database.RedisClient.HGet("system", "deps:python") + if err != nil { + return list, err + } + + // 反序列化 + if err := json.Unmarshal([]byte(rawData), &list); err != nil { + return list, err + } + + // 如果为空，则从依赖源获取列表 + if len(list) == 0 { + UpdatePythonDepList() + } + + return list, nil +} + +// 从Python依赖源获取依赖列表并返回 +func FetchPythonDepList() ([]string, error) { + // 依赖URL + url := "https://pypi.tuna.tsinghua.edu.cn/simple" + + // 输出列表 + var list []string + + // 请求URL + res, err := req.Get(url) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return list, err + } + + // 获取响应数据 + text, err := res.ToString() + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return list, err + } + + // 从响应数据中提取依赖名 + regex := regexp.MustCompile("(.*)") + for _, line := range strings.Split(text, "\n") { + arr := regex.FindStringSubmatch(line) + if len(arr) < 2 { + continue + } + list = append(list, arr[1]) + } + + // 赋值给列表 + return list, nil +} + +// 更新Python依赖列表到Redis +func UpdatePythonDepList() { + // 从依赖源获取列表 + list, _ := FetchPythonDepList() + + // 序列化 + listBytes, err := json.Marshal(list) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return + } + + // 设置Redis + if err := database.RedisClient.HSet("system", "deps:python", string(listBytes)); err != nil { + log.Error(err.Error()) + debug.PrintStack() + return + } +} + +// 获取Python本地已安装的依赖列表 +func GetPythonLocalInstalledDepList(nodeId string) ([]entity.Dependency, error) { + var list []entity.Dependency + + lang := GetLangFromLangName(nodeId, constants.Python) + if !IsInstalledLang(nodeId, lang) { + return list, errors.New("python is not installed") + } + cmd := exec.Command("pip", "freeze") + outputBytes, err := cmd.Output() + if err != nil { + debug.PrintStack() + return list, err + } + + for _, line := range strings.Split(string(outputBytes), "\n") { + arr := strings.Split(line, "==") + if len(arr) < 2 { + continue + } + dep := entity.Dependency{ + Name: strings.ToLower(arr[0]), + Version: arr[1], + Installed: true, + } + list = append(list, dep) + } + + return list, nil +} + +// 获取Python远端依赖列表 +func GetPythonRemoteInstalledDepList(nodeId string) ([]entity.Dependency, error) { + depList, err := RpcClientGetInstalledDepList(nodeId, constants.Python) + if err != nil { + return depList, err + } + return depList, nil +} + +// 安装Python本地依赖 +func InstallPythonLocalDep(depName string) (string, error) { + // 依赖镜像URL + url := "https://pypi.tuna.tsinghua.edu.cn/simple" + + cmd := exec.Command("pip", "install", depName, "-i", url) + outputBytes, err := cmd.Output() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return fmt.Sprintf("error: %s", err.Error()), err + } + return string(outputBytes), nil +} + +// 获取Python远端依赖列表 +func InstallPythonRemoteDep(nodeId string, depName string) (string, error) { + output, err := RpcClientInstallDep(nodeId, constants.Python, depName) + if err != nil { + return output, err + } + return output, nil +} + +// 安装Python本地依赖 +func UninstallPythonLocalDep(depName string) (string, error) { + cmd := exec.Command("pip", "uninstall", "-y", depName) + outputBytes, err := cmd.Output() + if err != nil { + log.Errorf(string(outputBytes)) + log.Errorf(err.Error()) + debug.PrintStack() + return fmt.Sprintf("error: %s", err.Error()), err + } + return string(outputBytes), nil +} + +// 获取Python远端依赖列表 +func UninstallPythonRemoteDep(nodeId string, depName string) (string, error) { + output, err := RpcClientUninstallDep(nodeId, constants.Python, depName) + if err != nil { + return output, err + } + return output, nil +} + +// ============== +// Node.js +// ============== + +func InstallNodejsLocalLang() (string, error) { + cmd := exec.Command("/bin/sh", path.Join("scripts", "install-nodejs.sh")) + output, err := cmd.Output() + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return string(output), err + } + + // TODO: check if Node.js is installed successfully + + return string(output), nil +} + +// 获取Node.js远端依赖列表 +func InstallNodejsRemoteLang(nodeId string) (string, error) { + output, err := RpcClientInstallLang(nodeId, constants.Nodejs) + if err != nil { + return output, err + } + return output, nil +} + +// 获取Nodejs本地已安装的依赖列表 +func GetNodejsLocalInstalledDepList(nodeId string) ([]entity.Dependency, error) { + var list []entity.Dependency + + lang := GetLangFromLangName(nodeId, constants.Nodejs) + if !IsInstalledLang(nodeId, lang) { + return list, errors.New("nodejs is not installed") + } + cmd := exec.Command("npm", "ls", "-g", "--depth", "0") + outputBytes, _ := cmd.Output() + //if err != nil { + // log.Error("error: " + string(outputBytes)) + // debug.PrintStack() + // return list, err + //} + + regex := regexp.MustCompile("\\s(.*)@(.*)") + for _, line := range strings.Split(string(outputBytes), "\n") { + arr := regex.FindStringSubmatch(line) + if len(arr) < 3 { + continue + } + dep := entity.Dependency{ + Name: strings.ToLower(arr[1]), + Version: arr[2], + Installed: true, + } + list = append(list, dep) + } + + return list, nil +} + +// 获取Nodejs远端依赖列表 +func GetNodejsRemoteInstalledDepList(nodeId string) ([]entity.Dependency, error) { + depList, err := RpcClientGetInstalledDepList(nodeId, constants.Nodejs) + if err != nil { + return depList, err + } + return depList, nil +} + +// 安装Nodejs本地依赖 +func InstallNodejsLocalDep(depName string) (string, error) { + // 依赖镜像URL + url := "https://registry.npm.taobao.org" + + cmd := exec.Command("npm", "install", depName, "-g", "--registry", url) + outputBytes, err := cmd.Output() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return fmt.Sprintf("error: %s", err.Error()), err + } + return string(outputBytes), nil +} + +// 获取Nodejs远端依赖列表 +func InstallNodejsRemoteDep(nodeId string, depName string) (string, error) { + output, err := RpcClientInstallDep(nodeId, constants.Nodejs, depName) + if err != nil { + return output, err + } + return output, nil +} + +// 安装Nodejs本地依赖 +func UninstallNodejsLocalDep(depName string) (string, error) { + cmd := exec.Command("npm", "uninstall", depName, "-g") + outputBytes, err := cmd.Output() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return fmt.Sprintf("error: %s", err.Error()), err + } + return string(outputBytes), nil +} + +// 获取Nodejs远端依赖列表 +func UninstallNodejsRemoteDep(nodeId string, depName string) (string, error) { + output, err := RpcClientUninstallDep(nodeId, constants.Nodejs, depName) + if err != nil { + return output, err + } + return output, nil +} + +// 获取Nodejs本地依赖列表 +func GetNodejsDepList(nodeId string, searchDepName string) (depList []entity.Dependency, err error) { + // 执行shell命令 + cmd := exec.Command("npm", "search", "--json", searchDepName) + outputBytes, _ := cmd.Output() + + // 获取已安装依赖列表 + var installedDepList []entity.Dependency + if IsMasterNode(nodeId) { + installedDepList, err = GetNodejsLocalInstalledDepList(nodeId) + if err != nil { + return depList, err + } + } else { + installedDepList, err = GetNodejsRemoteInstalledDepList(nodeId) + if err != nil { + return depList, err + } + } + + // 反序列化 + if err := json.Unmarshal(outputBytes, &depList); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return depList, err + } + + // 遍历安装列表 + for i, dep := range depList { + depList[i].Installed = IsInstalledDep(installedDepList, dep) + } + + return depList, nil +} diff --git a/backend/services/task.go b/backend/services/task.go index 9e6fdbc8..c0aa8fc8 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -19,6 +19,7 @@ import ( "runtime" "runtime/debug" "strconv" + "strings" "sync" "syscall" "time" @@ -104,6 +105,17 @@ func AssignTask(task model.Task) error { // 设置环境变量 func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exec.Cmd { + // 默认把Node.js的全局node_modules加入环境变量 + envPath := os.Getenv("PATH") + for _, _path := range strings.Split(envPath, ":") { + if strings.Contains(_path, "/.nvm/versions/node/") { + pathNodeModules := strings.Replace(_path, "/bin", "/lib/node_modules", -1) + _ = os.Setenv("PATH", pathNodeModules+":"+envPath) + _ = os.Setenv("NODE_PATH", pathNodeModules) + break + } + } + // 默认环境变量 cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+taskId) cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+dataCol) diff --git a/backend/vendor/github.com/imroc/req/LICENSE b/backend/vendor/github.com/imroc/req/LICENSE new file mode 100644 index 00000000..8dada3ed --- /dev/null +++ b/backend/vendor/github.com/imroc/req/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/backend/vendor/github.com/imroc/req/README.md b/backend/vendor/github.com/imroc/req/README.md new file mode 100644 index 00000000..c41d5ae0 --- /dev/null +++ b/backend/vendor/github.com/imroc/req/README.md @@ -0,0 +1,302 @@ +# req +[![GoDoc](https://godoc.org/github.com/imroc/req?status.svg)](https://godoc.org/github.com/imroc/req) + +A golang http request library for humans + + + +Features +======== + +- Light weight +- Simple +- Easy play with JSON and XML +- Easy for debug and logging +- Easy file uploads and downloads +- Easy manage cookie +- Easy set up proxy +- Easy set timeout +- Easy customize http client + + +Document +======== +[中文](doc/README_cn.md) + + +Install +======= +``` sh +go get github.com/imroc/req +``` + +Overview +======= +`req` implements a friendly API over Go's existing `net/http` library. + +`Req` and `Resp` are two most important struct, you can think of `Req` as a client that initiate HTTP requests, `Resp` as a information container for the request and response. They all provide simple and convenient APIs that allows you to do a lot of things. +``` go +func (r *Req) Post(url string, v ...interface{}) (*Resp, error) +``` + +In most cases, only url is required, others are optional, like headers, params, files or body etc. + +There is a default `Req` object, all of its' public methods are wrapped by the `req` package, so you can also think of `req` package as a `Req` object +``` go +// use Req object to initiate requests. +r := req.New() +r.Get(url) + +// use req package to initiate request. +req.Get(url) +``` +You can use `req.New()` to create lots of `*Req` as client with independent configuration + +Examples +======= +[Basic](#Basic) +[Set Header](#Set-Header) +[Set Param](#Set-Param) +[Set Body](#Set-Body) +[Debug](#Debug) +[Output Format](#Format) +[ToJSON & ToXML](#ToJSON-ToXML) +[Get *http.Response](#Response) +[Upload](#Upload) +[Download](#Download) +[Cookie](#Cookie) +[Set Timeout](#Set-Timeout) +[Set Proxy](#Set-Proxy) +[Customize Client](#Customize-Client) + +## Basic +``` go +header := req.Header{ + "Accept": "application/json", + "Authorization": "Basic YWRtaW46YWRtaW4=", +} +param := req.Param{ + "name": "imroc", + "cmd": "add", +} +// only url is required, others are optional. +r, err = req.Post("http://foo.bar/api", header, param) +if err != nil { + log.Fatal(err) +} +r.ToJSON(&foo) // response => struct/map +log.Printf("%+v", r) // print info (try it, you may surprise) +``` + +## Set Header +Use `req.Header` (it is actually a `map[string]string`) +``` go +authHeader := req.Header{ + "Accept": "application/json", + "Authorization": "Basic YWRtaW46YWRtaW4=", +} +req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"}) +``` +use `http.Header` +``` go +header := make(http.Header) +header.Set("Accept", "application/json") +req.Get("https://www.baidu.com", header) +``` + +## Set Param +Use `req.Param` (it is actually a `map[string]interface{}`) +``` go +param := req.Param{ + "id": "imroc", + "pwd": "roc", +} +req.Get("http://foo.bar/api", param) // http://foo.bar/api?id=imroc&pwd=roc +req.Post(url, param) // body => id=imroc&pwd=roc +``` +use `req.QueryParam` force to append params to the url (it is also actually a `map[string]interface{}`) +``` go +req.Post("http://foo.bar/api", req.Param{"name": "roc", "age": "22"}, req.QueryParam{"access_token": "fedledGF9Hg9ehTU"}) +/* +POST /api?access_token=fedledGF9Hg9ehTU HTTP/1.1 +Host: foo.bar +User-Agent: Go-http-client/1.1 +Content-Length: 15 +Content-Type: application/x-www-form-urlencoded;charset=UTF-8 +Accept-Encoding: gzip + +age=22&name=roc +*/ +``` + +## Set Body +Put `string`, `[]byte` and `io.Reader` as body directly. +``` go +req.Post(url, "id=roc&cmd=query") +``` +Put object as xml or json body (add `Content-Type` header automatically) +``` go +req.Post(url, req.BodyJSON(&foo)) +req.Post(url, req.BodyXML(&bar)) +``` + +## Debug +Set global variable `req.Debug` to true, it will print detail infomation for every request. +``` go +req.Debug = true +req.Post("http://localhost/test" "hi") +``` +![post](doc/post.png) + +## Output Format +You can use different kind of output format to log the request and response infomation in your log file in defferent scenarios. For example, use `%+v` output format in the development phase, it allows you to observe the details. Use `%v` or `%-v` output format in production phase, just log the information necessarily. + +### `%+v` or `%+s` +Output in detail +``` go +r, _ := req.Post(url, header, param) +log.Printf("%+v", r) // output the same format as Debug is enabled +``` + +### `%v` or `%s` +Output in simple way (default format) +``` go +r, _ := req.Get(url, param) +log.Printf("%v\n", r) // GET http://foo.bar/api?name=roc&cmd=add {"code":"0","msg":"success"} +log.Prinln(r) // smae as above +``` + +### `%-v` or `%-s` +Output in simple way and keep all in one line (request body or response body may have multiple lines, this format will replace `"\r"` or `"\n"` with `" "`, it's useful when doing some search in your log file) + +### Flag +You can call `SetFlags` to control the output content, decide which pieces can be output. +``` go +const ( + LreqHead = 1 << iota // output request head (request line and request header) + LreqBody // output request body + LrespHead // output response head (response line and response header) + LrespBody // output response body + Lcost // output time costed by the request + LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody +) +``` +``` go +req.SetFlags(req.LreqHead | req.LreqBody | req.LrespHead) +``` + +### Monitoring time consuming +``` go +req.SetFlags(req.LstdFlags | req.Lcost) // output format add time costed by request +r,_ := req.Get(url) +log.Println(r) // http://foo.bar/api 3.260802ms {"code":0 "msg":"success"} +if r.Cost() > 3 * time.Second { // check cost + log.Println("WARN: slow request:", r) +} +``` + +## ToJSON & ToXML +``` go +r, _ := req.Get(url) +r.ToJSON(&foo) +r, _ = req.Post(url, req.BodyXML(&bar)) +r.ToXML(&baz) +``` + +## Get *http.Response +```go +// func (r *Req) Response() *http.Response +r, _ := req.Get(url) +resp := r.Response() +fmt.Println(resp.StatusCode) +``` + +## Upload +Use `req.File` to match files +``` go +req.Post(url, req.File("imroc.png"), req.File("/Users/roc/Pictures/*.png")) +``` +Use `req.FileUpload` to fully control +``` go +file, _ := os.Open("imroc.png") +req.Post(url, req.FileUpload{ + File: file, + FieldName: "file", // FieldName is form field name + FileName: "avatar.png", //Filename is the name of the file that you wish to upload. We use this to guess the mimetype as well as pass it onto the server +}) +``` +Use `req.UploadProgress` to listen upload progress +```go +progress := func(current, total int64) { + fmt.Println(float32(current)/float32(total)*100, "%") +} +req.Post(url, req.File("/Users/roc/Pictures/*.png"), req.UploadProgress(progress)) +fmt.Println("upload complete") +``` + +## Download +``` go +r, _ := req.Get(url) +r.ToFile("imroc.png") +``` +Use `req.DownloadProgress` to listen download progress +```go +progress := func(current, total int64) { + fmt.Println(float32(current)/float32(total)*100, "%") +} +r, _ := req.Get(url, req.DownloadProgress(progress)) +r.ToFile("hello.mp4") +fmt.Println("download complete") +``` + +## Cookie +By default, the underlying `*http.Client` will manage your cookie(send cookie header to server automatically if server has set a cookie for you), you can disable it by calling this function : +``` go +req.EnableCookie(false) +``` +and you can set cookie in request just using `*http.Cookie` +``` go +cookie := new(http.Cookie) +// ...... +req.Get(url, cookie) +``` + +## Set Timeout +``` go +req.SetTimeout(50 * time.Second) +``` + +## Set Proxy +By default, req use proxy from system environment if `http_proxy` or `https_proxy` is specified, you can set a custom proxy or disable it by set `nil` +``` go +req.SetProxy(func(r *http.Request) (*url.URL, error) { + if strings.Contains(r.URL.Hostname(), "google") { + return url.Parse("http://my.vpn.com:23456") + } + return nil, nil +}) +``` +Set a simple proxy (use fixed proxy url for every request) +``` go +req.SetProxyUrl("http://my.proxy.com:23456") +``` + +## Customize Client +Use `SetClient` to change the default underlying `*http.Client` +``` go +req.SetClient(client) +``` +Specify independent http client for some requests +``` go +client := &http.Client{Timeout: 30 * time.Second} +req.Get(url, client) +``` +Change some properties of default client you want +``` go +req.Client().Jar, _ = cookiejar.New(nil) +trans, _ := req.Client().Transport.(*http.Transport) +trans.MaxIdleConns = 20 +trans.TLSHandshakeTimeout = 20 * time.Second +trans.DisableKeepAlives = true +trans.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} +``` diff --git a/backend/vendor/github.com/imroc/req/dump.go b/backend/vendor/github.com/imroc/req/dump.go new file mode 100644 index 00000000..ce6d3a5b --- /dev/null +++ b/backend/vendor/github.com/imroc/req/dump.go @@ -0,0 +1,216 @@ +package req + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "time" +) + +// Debug enable debug mode if set to true +var Debug bool + +// dumpConn is a net.Conn which writes to Writer and reads from Reader +type dumpConn struct { + io.Writer + io.Reader +} + +func (c *dumpConn) Close() error { return nil } +func (c *dumpConn) LocalAddr() net.Addr { return nil } +func (c *dumpConn) RemoteAddr() net.Addr { return nil } +func (c *dumpConn) SetDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetReadDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetWriteDeadline(t time.Time) error { return nil } + +// delegateReader is a reader that delegates to another reader, +// once it arrives on a channel. +type delegateReader struct { + c chan io.Reader + r io.Reader // nil until received from c +} + +func (r *delegateReader) Read(p []byte) (int, error) { + if r.r == nil { + r.r = <-r.c + } + return r.r.Read(p) +} + +type dummyBody struct { + N int + off int +} + +func (d *dummyBody) Read(p []byte) (n int, err error) { + if d.N <= 0 { + err = io.EOF + return + } + left := d.N - d.off + if left <= 0 { + err = io.EOF + return + } + + if l := len(p); l > 0 { + if l >= left { + n = left + err = io.EOF + } else { + n = l + } + d.off += n + for i := 0; i < n; i++ { + p[i] = '*' + } + } + + return +} + +func (d *dummyBody) Close() error { + return nil +} + +type dumpBuffer struct { + bytes.Buffer +} + +func (b *dumpBuffer) Write(p []byte) { + if b.Len() > 0 { + b.Buffer.WriteString("\r\n\r\n") + } + b.Buffer.Write(p) +} + +func (b *dumpBuffer) WriteString(s string) { + b.Write([]byte(s)) +} + +func (r *Resp) dumpRequest(dump *dumpBuffer) { + head := r.r.flag&LreqHead != 0 + body := r.r.flag&LreqBody != 0 + + if head { + r.dumpReqHead(dump) + } + if body { + if r.multipartHelper != nil { + dump.Write(r.multipartHelper.Dump()) + } else if len(r.reqBody) > 0 { + dump.Write(r.reqBody) + } + } +} + +func (r *Resp) dumpReqHead(dump *dumpBuffer) { + reqSend := new(http.Request) + *reqSend = *r.req + if reqSend.URL.Scheme == "https" { + reqSend.URL = new(url.URL) + *reqSend.URL = *r.req.URL + reqSend.URL.Scheme = "http" + } + + if reqSend.ContentLength > 0 { + reqSend.Body = &dummyBody{N: int(reqSend.ContentLength)} + } else { + reqSend.Body = &dummyBody{N: 1} + } + + // Use the actual Transport code to record what we would send + // on the wire, but not using TCP. Use a Transport with a + // custom dialer that returns a fake net.Conn that waits + // for the full input (and recording it), and then responds + // with a dummy response. + var buf bytes.Buffer // records the output + pr, pw := io.Pipe() + defer pw.Close() + dr := &delegateReader{c: make(chan io.Reader)} + + t := &http.Transport{ + Dial: func(net, addr string) (net.Conn, error) { + return &dumpConn{io.MultiWriter(&buf, pw), dr}, nil + }, + } + defer t.CloseIdleConnections() + + client := new(http.Client) + *client = *r.client + client.Transport = t + + // Wait for the request before replying with a dummy response: + go func() { + req, err := http.ReadRequest(bufio.NewReader(pr)) + if err == nil { + // Ensure all the body is read; otherwise + // we'll get a partial dump. + io.Copy(ioutil.Discard, req.Body) + req.Body.Close() + } + + dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n") + pr.Close() + }() + + _, err := client.Do(reqSend) + if err != nil { + dump.WriteString(err.Error()) + } else { + reqDump := buf.Bytes() + if i := bytes.Index(reqDump, []byte("\r\n\r\n")); i >= 0 { + reqDump = reqDump[:i] + } + dump.Write(reqDump) + } +} + +func (r *Resp) dumpResponse(dump *dumpBuffer) { + head := r.r.flag&LrespHead != 0 + body := r.r.flag&LrespBody != 0 + if head { + respDump, err := httputil.DumpResponse(r.resp, false) + if err != nil { + dump.WriteString(err.Error()) + } else { + if i := bytes.Index(respDump, []byte("\r\n\r\n")); i >= 0 { + respDump = respDump[:i] + } + dump.Write(respDump) + } + } + if body && len(r.Bytes()) > 0 { + dump.Write(r.Bytes()) + } +} + +// Cost return the time cost of the request +func (r *Resp) Cost() time.Duration { + return r.cost +} + +// Dump dump the request +func (r *Resp) Dump() string { + dump := new(dumpBuffer) + if r.r.flag&Lcost != 0 { + dump.WriteString(fmt.Sprint(r.cost)) + } + r.dumpRequest(dump) + l := dump.Len() + if l > 0 { + dump.WriteString("=================================") + l = dump.Len() + } + + r.dumpResponse(dump) + + return dump.String() +} diff --git a/backend/vendor/github.com/imroc/req/req.go b/backend/vendor/github.com/imroc/req/req.go new file mode 100644 index 00000000..d1b3e712 --- /dev/null +++ b/backend/vendor/github.com/imroc/req/req.go @@ -0,0 +1,688 @@ +package req + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +// default *Req +var std = New() + +// flags to decide which part can be outputed +const ( + LreqHead = 1 << iota // output request head (request line and request header) + LreqBody // output request body + LrespHead // output response head (response line and response header) + LrespBody // output response body + Lcost // output time costed by the request + LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody +) + +// Header represents http request header +type Header map[string]string + +func (h Header) Clone() Header { + if h == nil { + return nil + } + hh := Header{} + for k, v := range h { + hh[k] = v + } + return hh +} + +// Param represents http request param +type Param map[string]interface{} + +// QueryParam is used to force append http request param to the uri +type QueryParam map[string]interface{} + +// Host is used for set request's Host +type Host string + +// FileUpload represents a file to upload +type FileUpload struct { + // filename in multipart form. + FileName string + // form field name + FieldName string + // file to uplaod, required + File io.ReadCloser +} + +type DownloadProgress func(current, total int64) + +type UploadProgress func(current, total int64) + +// File upload files matching the name pattern such as +// /usr/*/bin/go* (assuming the Separator is '/') +func File(patterns ...string) interface{} { + matches := []string{} + for _, pattern := range patterns { + m, err := filepath.Glob(pattern) + if err != nil { + return err + } + matches = append(matches, m...) + } + if len(matches) == 0 { + return errors.New("req: no file have been matched") + } + uploads := []FileUpload{} + for _, match := range matches { + if s, e := os.Stat(match); e != nil || s.IsDir() { + continue + } + file, _ := os.Open(match) + uploads = append(uploads, FileUpload{ + File: file, + FileName: filepath.Base(match), + FieldName: "media", + }) + } + + return uploads +} + +type bodyJson struct { + v interface{} +} + +type bodyXml struct { + v interface{} +} + +// BodyJSON make the object be encoded in json format and set it to the request body +func BodyJSON(v interface{}) *bodyJson { + return &bodyJson{v: v} +} + +// BodyXML make the object be encoded in xml format and set it to the request body +func BodyXML(v interface{}) *bodyXml { + return &bodyXml{v: v} +} + +// Req is a convenient client for initiating requests +type Req struct { + client *http.Client + jsonEncOpts *jsonEncOpts + xmlEncOpts *xmlEncOpts + flag int +} + +// New create a new *Req +func New() *Req { + return &Req{flag: LstdFlags} +} + +type param struct { + url.Values +} + +func (p *param) getValues() url.Values { + if p.Values == nil { + p.Values = make(url.Values) + } + return p.Values +} + +func (p *param) Copy(pp param) { + if pp.Values == nil { + return + } + vs := p.getValues() + for key, values := range pp.Values { + for _, value := range values { + vs.Add(key, value) + } + } +} +func (p *param) Adds(m map[string]interface{}) { + if len(m) == 0 { + return + } + vs := p.getValues() + for k, v := range m { + vs.Add(k, fmt.Sprint(v)) + } +} + +func (p *param) Empty() bool { + return p.Values == nil +} + +// Do execute a http request with sepecify method and url, +// and it can also have some optional params, depending on your needs. +func (r *Req) Do(method, rawurl string, vs ...interface{}) (resp *Resp, err error) { + if rawurl == "" { + return nil, errors.New("req: url not specified") + } + req := &http.Request{ + Method: method, + Header: make(http.Header), + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + } + resp = &Resp{req: req, r: r} + + var queryParam param + var formParam param + var uploads []FileUpload + var uploadProgress UploadProgress + var progress func(int64, int64) + var delayedFunc []func() + var lastFunc []func() + + for _, v := range vs { + switch vv := v.(type) { + case Header: + for key, value := range vv { + req.Header.Add(key, value) + } + case http.Header: + for key, values := range vv { + for _, value := range values { + req.Header.Add(key, value) + } + } + case *bodyJson: + fn, err := setBodyJson(req, resp, r.jsonEncOpts, vv.v) + if err != nil { + return nil, err + } + delayedFunc = append(delayedFunc, fn) + case *bodyXml: + fn, err := setBodyXml(req, resp, r.xmlEncOpts, vv.v) + if err != nil { + return nil, err + } + delayedFunc = append(delayedFunc, fn) + case url.Values: + p := param{vv} + if method == "GET" || method == "HEAD" { + queryParam.Copy(p) + } else { + formParam.Copy(p) + } + case Param: + if method == "GET" || method == "HEAD" { + queryParam.Adds(vv) + } else { + formParam.Adds(vv) + } + case QueryParam: + queryParam.Adds(vv) + case string: + setBodyBytes(req, resp, []byte(vv)) + case []byte: + setBodyBytes(req, resp, vv) + case bytes.Buffer: + setBodyBytes(req, resp, vv.Bytes()) + case *http.Client: + resp.client = vv + case FileUpload: + uploads = append(uploads, vv) + case []FileUpload: + uploads = append(uploads, vv...) + case *http.Cookie: + req.AddCookie(vv) + case Host: + req.Host = string(vv) + case io.Reader: + fn := setBodyReader(req, resp, vv) + lastFunc = append(lastFunc, fn) + case UploadProgress: + uploadProgress = vv + case DownloadProgress: + resp.downloadProgress = vv + case func(int64, int64): + progress = vv + case context.Context: + req = req.WithContext(vv) + resp.req = req + case error: + return nil, vv + } + } + + if length := req.Header.Get("Content-Length"); length != "" { + if l, err := strconv.ParseInt(length, 10, 64); err == nil { + req.ContentLength = l + } + } + + if len(uploads) > 0 && (req.Method == "POST" || req.Method == "PUT") { // multipart + var up UploadProgress + if uploadProgress != nil { + up = uploadProgress + } else if progress != nil { + up = UploadProgress(progress) + } + multipartHelper := &multipartHelper{ + form: formParam.Values, + uploads: uploads, + uploadProgress: up, + } + multipartHelper.Upload(req) + resp.multipartHelper = multipartHelper + } else { + if progress != nil { + resp.downloadProgress = DownloadProgress(progress) + } + if !formParam.Empty() { + if req.Body != nil { + queryParam.Copy(formParam) + } else { + setBodyBytes(req, resp, []byte(formParam.Encode())) + setContentType(req, "application/x-www-form-urlencoded; charset=UTF-8") + } + } + } + + if !queryParam.Empty() { + paramStr := queryParam.Encode() + if strings.IndexByte(rawurl, '?') == -1 { + rawurl = rawurl + "?" + paramStr + } else { + rawurl = rawurl + "&" + paramStr + } + } + + u, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + req.URL = u + + if host := req.Header.Get("Host"); host != "" { + req.Host = host + } + + for _, fn := range delayedFunc { + fn() + } + + if resp.client == nil { + resp.client = r.Client() + } + + var response *http.Response + if r.flag&Lcost != 0 { + before := time.Now() + response, err = resp.client.Do(req) + after := time.Now() + resp.cost = after.Sub(before) + } else { + response, err = resp.client.Do(req) + } + if err != nil { + return nil, err + } + + for _, fn := range lastFunc { + fn() + } + + resp.resp = response + + if _, ok := resp.client.Transport.(*http.Transport); ok && response.Header.Get("Content-Encoding") == "gzip" && req.Header.Get("Accept-Encoding") != "" { + body, err := gzip.NewReader(response.Body) + if err != nil { + return nil, err + } + response.Body = body + } + + // output detail if Debug is enabled + if Debug { + fmt.Println(resp.Dump()) + } + return +} + +func setBodyBytes(req *http.Request, resp *Resp, data []byte) { + resp.reqBody = data + req.Body = ioutil.NopCloser(bytes.NewReader(data)) + req.ContentLength = int64(len(data)) +} + +func setBodyJson(req *http.Request, resp *Resp, opts *jsonEncOpts, v interface{}) (func(), error) { + var data []byte + switch vv := v.(type) { + case string: + data = []byte(vv) + case []byte: + data = vv + case *bytes.Buffer: + data = vv.Bytes() + default: + if opts != nil { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent(opts.indentPrefix, opts.indentValue) + enc.SetEscapeHTML(opts.escapeHTML) + err := enc.Encode(v) + if err != nil { + return nil, err + } + data = buf.Bytes() + } else { + var err error + data, err = json.Marshal(v) + if err != nil { + return nil, err + } + } + } + setBodyBytes(req, resp, data) + delayedFunc := func() { + setContentType(req, "application/json; charset=UTF-8") + } + return delayedFunc, nil +} + +func setBodyXml(req *http.Request, resp *Resp, opts *xmlEncOpts, v interface{}) (func(), error) { + var data []byte + switch vv := v.(type) { + case string: + data = []byte(vv) + case []byte: + data = vv + case *bytes.Buffer: + data = vv.Bytes() + default: + if opts != nil { + var buf bytes.Buffer + enc := xml.NewEncoder(&buf) + enc.Indent(opts.prefix, opts.indent) + err := enc.Encode(v) + if err != nil { + return nil, err + } + data = buf.Bytes() + } else { + var err error + data, err = xml.Marshal(v) + if err != nil { + return nil, err + } + } + } + setBodyBytes(req, resp, data) + delayedFunc := func() { + setContentType(req, "application/xml; charset=UTF-8") + } + return delayedFunc, nil +} + +func setContentType(req *http.Request, contentType string) { + if req.Header.Get("Content-Type") == "" { + req.Header.Set("Content-Type", contentType) + } +} + +func setBodyReader(req *http.Request, resp *Resp, rd io.Reader) func() { + var rc io.ReadCloser + switch r := rd.(type) { + case *os.File: + stat, err := r.Stat() + if err == nil { + req.ContentLength = stat.Size() + } + rc = r + + case io.ReadCloser: + rc = r + default: + rc = ioutil.NopCloser(rd) + } + bw := &bodyWrapper{ + ReadCloser: rc, + limit: 102400, + } + req.Body = bw + lastFunc := func() { + resp.reqBody = bw.buf.Bytes() + } + return lastFunc +} + +type bodyWrapper struct { + io.ReadCloser + buf bytes.Buffer + limit int +} + +func (b *bodyWrapper) Read(p []byte) (n int, err error) { + n, err = b.ReadCloser.Read(p) + if left := b.limit - b.buf.Len(); left > 0 && n > 0 { + if n <= left { + b.buf.Write(p[:n]) + } else { + b.buf.Write(p[:left]) + } + } + return +} + +type multipartHelper struct { + form url.Values + uploads []FileUpload + dump []byte + uploadProgress UploadProgress +} + +func (m *multipartHelper) Upload(req *http.Request) { + pr, pw := io.Pipe() + bodyWriter := multipart.NewWriter(pw) + go func() { + for key, values := range m.form { + for _, value := range values { + bodyWriter.WriteField(key, value) + } + } + var upload func(io.Writer, io.Reader) error + if m.uploadProgress != nil { + var total int64 + for _, up := range m.uploads { + if file, ok := up.File.(*os.File); ok { + stat, err := file.Stat() + if err != nil { + continue + } + total += stat.Size() + } + } + var current int64 + buf := make([]byte, 1024) + var lastTime time.Time + upload = func(w io.Writer, r io.Reader) error { + for { + n, err := r.Read(buf) + if n > 0 { + _, _err := w.Write(buf[:n]) + if _err != nil { + return _err + } + current += int64(n) + if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond { + lastTime = now + m.uploadProgress(current, total) + } + } + if err == io.EOF { + return nil + } + if err != nil { + return err + } + } + } + } + + i := 0 + for _, up := range m.uploads { + if up.FieldName == "" { + i++ + up.FieldName = "file" + strconv.Itoa(i) + } + fileWriter, err := bodyWriter.CreateFormFile(up.FieldName, up.FileName) + if err != nil { + continue + } + //iocopy + if upload == nil { + io.Copy(fileWriter, up.File) + } else { + if _, ok := up.File.(*os.File); ok { + upload(fileWriter, up.File) + } else { + io.Copy(fileWriter, up.File) + } + } + up.File.Close() + } + bodyWriter.Close() + pw.Close() + }() + req.Header.Set("Content-Type", bodyWriter.FormDataContentType()) + req.Body = ioutil.NopCloser(pr) +} + +func (m *multipartHelper) Dump() []byte { + if m.dump != nil { + return m.dump + } + var buf bytes.Buffer + bodyWriter := multipart.NewWriter(&buf) + for key, values := range m.form { + for _, value := range values { + m.writeField(bodyWriter, key, value) + } + } + for _, up := range m.uploads { + m.writeFile(bodyWriter, up.FieldName, up.FileName) + } + bodyWriter.Close() + m.dump = buf.Bytes() + return m.dump +} + +func (m *multipartHelper) writeField(w *multipart.Writer, fieldname, value string) error { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"`, fieldname)) + p, err := w.CreatePart(h) + if err != nil { + return err + } + _, err = p.Write([]byte(value)) + return err +} + +func (m *multipartHelper) writeFile(w *multipart.Writer, fieldname, filename string) error { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + fieldname, filename)) + h.Set("Content-Type", "application/octet-stream") + p, err := w.CreatePart(h) + if err != nil { + return err + } + _, err = p.Write([]byte("******")) + return err +} + +// Get execute a http GET request +func (r *Req) Get(url string, v ...interface{}) (*Resp, error) { + return r.Do("GET", url, v...) +} + +// Post execute a http POST request +func (r *Req) Post(url string, v ...interface{}) (*Resp, error) { + return r.Do("POST", url, v...) +} + +// Put execute a http PUT request +func (r *Req) Put(url string, v ...interface{}) (*Resp, error) { + return r.Do("PUT", url, v...) +} + +// Patch execute a http PATCH request +func (r *Req) Patch(url string, v ...interface{}) (*Resp, error) { + return r.Do("PATCH", url, v...) +} + +// Delete execute a http DELETE request +func (r *Req) Delete(url string, v ...interface{}) (*Resp, error) { + return r.Do("DELETE", url, v...) +} + +// Head execute a http HEAD request +func (r *Req) Head(url string, v ...interface{}) (*Resp, error) { + return r.Do("HEAD", url, v...) +} + +// Options execute a http OPTIONS request +func (r *Req) Options(url string, v ...interface{}) (*Resp, error) { + return r.Do("OPTIONS", url, v...) +} + +// Get execute a http GET request +func Get(url string, v ...interface{}) (*Resp, error) { + return std.Get(url, v...) +} + +// Post execute a http POST request +func Post(url string, v ...interface{}) (*Resp, error) { + return std.Post(url, v...) +} + +// Put execute a http PUT request +func Put(url string, v ...interface{}) (*Resp, error) { + return std.Put(url, v...) +} + +// Head execute a http HEAD request +func Head(url string, v ...interface{}) (*Resp, error) { + return std.Head(url, v...) +} + +// Options execute a http OPTIONS request +func Options(url string, v ...interface{}) (*Resp, error) { + return std.Options(url, v...) +} + +// Delete execute a http DELETE request +func Delete(url string, v ...interface{}) (*Resp, error) { + return std.Delete(url, v...) +} + +// Patch execute a http PATCH request +func Patch(url string, v ...interface{}) (*Resp, error) { + return std.Patch(url, v...) +} + +// Do execute request. +func Do(method, url string, v ...interface{}) (*Resp, error) { + return std.Do(method, url, v...) +} diff --git a/backend/vendor/github.com/imroc/req/resp.go b/backend/vendor/github.com/imroc/req/resp.go new file mode 100644 index 00000000..eb56b1bd --- /dev/null +++ b/backend/vendor/github.com/imroc/req/resp.go @@ -0,0 +1,215 @@ +package req + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "regexp" + "time" +) + +// Resp represents a request with it's response +type Resp struct { + r *Req + req *http.Request + resp *http.Response + client *http.Client + cost time.Duration + *multipartHelper + reqBody []byte + respBody []byte + downloadProgress DownloadProgress + err error // delayed error +} + +// Request returns *http.Request +func (r *Resp) Request() *http.Request { + return r.req +} + +// Response returns *http.Response +func (r *Resp) Response() *http.Response { + return r.resp +} + +// Bytes returns response body as []byte +func (r *Resp) Bytes() []byte { + data, _ := r.ToBytes() + return data +} + +// ToBytes returns response body as []byte, +// return error if error happend when reading +// the response body +func (r *Resp) ToBytes() ([]byte, error) { + if r.err != nil { + return nil, r.err + } + if r.respBody != nil { + return r.respBody, nil + } + defer r.resp.Body.Close() + respBody, err := ioutil.ReadAll(r.resp.Body) + if err != nil { + r.err = err + return nil, err + } + r.respBody = respBody + return r.respBody, nil +} + +// String returns response body as string +func (r *Resp) String() string { + data, _ := r.ToBytes() + return string(data) +} + +// ToString returns response body as string, +// return error if error happend when reading +// the response body +func (r *Resp) ToString() (string, error) { + data, err := r.ToBytes() + return string(data), err +} + +// ToJSON convert json response body to struct or map +func (r *Resp) ToJSON(v interface{}) error { + data, err := r.ToBytes() + if err != nil { + return err + } + return json.Unmarshal(data, v) +} + +// ToXML convert xml response body to struct or map +func (r *Resp) ToXML(v interface{}) error { + data, err := r.ToBytes() + if err != nil { + return err + } + return xml.Unmarshal(data, v) +} + +// ToFile download the response body to file with optional download callback +func (r *Resp) ToFile(name string) error { + //TODO set name to the suffix of url path if name == "" + file, err := os.Create(name) + if err != nil { + return err + } + defer file.Close() + + if r.respBody != nil { + _, err = file.Write(r.respBody) + return err + } + + if r.downloadProgress != nil && r.resp.ContentLength > 0 { + return r.download(file) + } + + defer r.resp.Body.Close() + _, err = io.Copy(file, r.resp.Body) + return err +} + +func (r *Resp) download(file *os.File) error { + p := make([]byte, 1024) + b := r.resp.Body + defer b.Close() + total := r.resp.ContentLength + var current int64 + var lastTime time.Time + for { + l, err := b.Read(p) + if l > 0 { + _, _err := file.Write(p[:l]) + if _err != nil { + return _err + } + current += int64(l) + if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond { + lastTime = now + r.downloadProgress(current, total) + } + } + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + +var regNewline = regexp.MustCompile(`\n|\r`) + +func (r *Resp) autoFormat(s fmt.State) { + req := r.req + if r.r.flag&Lcost != 0 { + fmt.Fprint(s, req.Method, " ", req.URL.String(), " ", r.cost) + } else { + fmt.Fprint(s, req.Method, " ", req.URL.String()) + } + + // test if it is should be outputed pretty + var pretty bool + var parts []string + addPart := func(part string) { + if part == "" { + return + } + parts = append(parts, part) + if !pretty && regNewline.MatchString(part) { + pretty = true + } + } + if r.r.flag&LreqBody != 0 { // request body + addPart(string(r.reqBody)) + } + if r.r.flag&LrespBody != 0 { // response body + addPart(r.String()) + } + + for _, part := range parts { + if pretty { + fmt.Fprint(s, "\n") + } + fmt.Fprint(s, " ", part) + } +} + +func (r *Resp) miniFormat(s fmt.State) { + req := r.req + if r.r.flag&Lcost != 0 { + fmt.Fprint(s, req.Method, " ", req.URL.String(), " ", r.cost) + } else { + fmt.Fprint(s, req.Method, " ", req.URL.String()) + } + if r.r.flag&LreqBody != 0 && len(r.reqBody) > 0 { // request body + str := regNewline.ReplaceAllString(string(r.reqBody), " ") + fmt.Fprint(s, " ", str) + } + if r.r.flag&LrespBody != 0 && r.String() != "" { // response body + str := regNewline.ReplaceAllString(r.String(), " ") + fmt.Fprint(s, " ", str) + } +} + +// Format fort the response +func (r *Resp) Format(s fmt.State, verb rune) { + if r == nil || r.req == nil { + return + } + if s.Flag('+') { // include header and format pretty. + fmt.Fprint(s, r.Dump()) + } else if s.Flag('-') { // keep all informations in one line. + r.miniFormat(s) + } else { // auto + r.autoFormat(s) + } +} diff --git a/backend/vendor/github.com/imroc/req/setting.go b/backend/vendor/github.com/imroc/req/setting.go new file mode 100644 index 00000000..74235f37 --- /dev/null +++ b/backend/vendor/github.com/imroc/req/setting.go @@ -0,0 +1,236 @@ +package req + +import ( + "crypto/tls" + "errors" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "time" +) + +// create a default client +func newClient() *http.Client { + jar, _ := cookiejar.New(nil) + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + return &http.Client{ + Jar: jar, + Transport: transport, + Timeout: 2 * time.Minute, + } +} + +// Client return the default underlying http client +func (r *Req) Client() *http.Client { + if r.client == nil { + r.client = newClient() + } + return r.client +} + +// Client return the default underlying http client +func Client() *http.Client { + return std.Client() +} + +// SetClient sets the underlying http.Client. +func (r *Req) SetClient(client *http.Client) { + r.client = client // use default if client == nil +} + +// SetClient sets the default http.Client for requests. +func SetClient(client *http.Client) { + std.SetClient(client) +} + +// SetFlags control display format of *Resp +func (r *Req) SetFlags(flags int) { + r.flag = flags +} + +// SetFlags control display format of *Resp +func SetFlags(flags int) { + std.SetFlags(flags) +} + +// Flags return output format for the *Resp +func (r *Req) Flags() int { + return r.flag +} + +// Flags return output format for the *Resp +func Flags() int { + return std.Flags() +} + +func (r *Req) getTransport() *http.Transport { + trans, _ := r.Client().Transport.(*http.Transport) + return trans +} + +// EnableInsecureTLS allows insecure https +func (r *Req) EnableInsecureTLS(enable bool) { + trans := r.getTransport() + if trans == nil { + return + } + if trans.TLSClientConfig == nil { + trans.TLSClientConfig = &tls.Config{} + } + trans.TLSClientConfig.InsecureSkipVerify = enable +} + +func EnableInsecureTLS(enable bool) { + std.EnableInsecureTLS(enable) +} + +// EnableCookieenable or disable cookie manager +func (r *Req) EnableCookie(enable bool) { + if enable { + jar, _ := cookiejar.New(nil) + r.Client().Jar = jar + } else { + r.Client().Jar = nil + } +} + +// EnableCookieenable or disable cookie manager +func EnableCookie(enable bool) { + std.EnableCookie(enable) +} + +// SetTimeout sets the timeout for every request +func (r *Req) SetTimeout(d time.Duration) { + r.Client().Timeout = d +} + +// SetTimeout sets the timeout for every request +func SetTimeout(d time.Duration) { + std.SetTimeout(d) +} + +// SetProxyUrl set the simple proxy with fixed proxy url +func (r *Req) SetProxyUrl(rawurl string) error { + trans := r.getTransport() + if trans == nil { + return errors.New("req: no transport") + } + u, err := url.Parse(rawurl) + if err != nil { + return err + } + trans.Proxy = http.ProxyURL(u) + return nil +} + +// SetProxyUrl set the simple proxy with fixed proxy url +func SetProxyUrl(rawurl string) error { + return std.SetProxyUrl(rawurl) +} + +// SetProxy sets the proxy for every request +func (r *Req) SetProxy(proxy func(*http.Request) (*url.URL, error)) error { + trans := r.getTransport() + if trans == nil { + return errors.New("req: no transport") + } + trans.Proxy = proxy + return nil +} + +// SetProxy sets the proxy for every request +func SetProxy(proxy func(*http.Request) (*url.URL, error)) error { + return std.SetProxy(proxy) +} + +type jsonEncOpts struct { + indentPrefix string + indentValue string + escapeHTML bool +} + +func (r *Req) getJSONEncOpts() *jsonEncOpts { + if r.jsonEncOpts == nil { + r.jsonEncOpts = &jsonEncOpts{escapeHTML: true} + } + return r.jsonEncOpts +} + +// SetJSONEscapeHTML specifies whether problematic HTML characters +// should be escaped inside JSON quoted strings. +// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e +// to avoid certain safety problems that can arise when embedding JSON in HTML. +// +// In non-HTML settings where the escaping interferes with the readability +// of the output, SetEscapeHTML(false) disables this behavior. +func (r *Req) SetJSONEscapeHTML(escape bool) { + opts := r.getJSONEncOpts() + opts.escapeHTML = escape +} + +// SetJSONEscapeHTML specifies whether problematic HTML characters +// should be escaped inside JSON quoted strings. +// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e +// to avoid certain safety problems that can arise when embedding JSON in HTML. +// +// In non-HTML settings where the escaping interferes with the readability +// of the output, SetEscapeHTML(false) disables this behavior. +func SetJSONEscapeHTML(escape bool) { + std.SetJSONEscapeHTML(escape) +} + +// SetJSONIndent instructs the encoder to format each subsequent encoded +// value as if indented by the package-level function Indent(dst, src, prefix, indent). +// Calling SetIndent("", "") disables indentation. +func (r *Req) SetJSONIndent(prefix, indent string) { + opts := r.getJSONEncOpts() + opts.indentPrefix = prefix + opts.indentValue = indent +} + +// SetJSONIndent instructs the encoder to format each subsequent encoded +// value as if indented by the package-level function Indent(dst, src, prefix, indent). +// Calling SetIndent("", "") disables indentation. +func SetJSONIndent(prefix, indent string) { + std.SetJSONIndent(prefix, indent) +} + +type xmlEncOpts struct { + prefix string + indent string +} + +func (r *Req) getXMLEncOpts() *xmlEncOpts { + if r.xmlEncOpts == nil { + r.xmlEncOpts = &xmlEncOpts{} + } + return r.xmlEncOpts +} + +// SetXMLIndent sets the encoder to generate XML in which each element +// begins on a new indented line that starts with prefix and is followed by +// one or more copies of indent according to the nesting depth. +func (r *Req) SetXMLIndent(prefix, indent string) { + opts := r.getXMLEncOpts() + opts.prefix = prefix + opts.indent = indent +} + +// SetXMLIndent sets the encoder to generate XML in which each element +// begins on a new indented line that starts with prefix and is followed by +// one or more copies of indent according to the nesting depth. +func SetXMLIndent(prefix, indent string) { + std.SetXMLIndent(prefix, indent) +} diff --git a/backend/vendor/github.com/json-iterator/go/Gopkg.lock b/backend/vendor/github.com/json-iterator/go/Gopkg.lock deleted file mode 100644 index c8a9fbb3..00000000 --- a/backend/vendor/github.com/json-iterator/go/Gopkg.lock +++ /dev/null @@ -1,21 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - name = "github.com/modern-go/concurrent" - packages = ["."] - revision = "e0a39a4cb4216ea8db28e22a69f4ec25610d513a" - version = "1.0.0" - -[[projects]] - name = "github.com/modern-go/reflect2" - packages = ["."] - revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" - version = "1.0.1" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "ea54a775e5a354cb015502d2e7aa4b74230fc77e894f34a838b268c25ec8eeb8" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/backend/vendor/github.com/modern-go/reflect2/Gopkg.lock b/backend/vendor/github.com/modern-go/reflect2/Gopkg.lock deleted file mode 100644 index 2a3a6989..00000000 --- a/backend/vendor/github.com/modern-go/reflect2/Gopkg.lock +++ /dev/null @@ -1,15 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - name = "github.com/modern-go/concurrent" - packages = ["."] - revision = "e0a39a4cb4216ea8db28e22a69f4ec25610d513a" - version = "1.0.0" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "daee8a88b3498b61c5640056665b8b9eea062006f5e596bbb6a3ed9119a11ec7" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index 99c90a5c..c6a30b46 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -40,6 +40,8 @@ github.com/hashicorp/hcl/hcl/token github.com/hashicorp/hcl/json/parser github.com/hashicorp/hcl/json/scanner github.com/hashicorp/hcl/json/token +# github.com/imroc/req v0.2.4 +github.com/imroc/req # github.com/json-iterator/go v1.1.6 github.com/json-iterator/go # github.com/jtolds/gls v4.20.0+incompatible diff --git a/docker-compose.yml b/docker-compose.yml index b4f36e86..5c059f95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,12 +8,15 @@ services: CRAWLAB_SERVER_MASTER: "Y" # whether to be master node 是否为主节点，主节点为 Y，工作节点为 N CRAWLAB_MONGO_HOST: "mongo" # MongoDB host address MongoDB 的地址，在 docker compose 网络中，直接引用服务名称 CRAWLAB_REDIS_ADDRESS: "redis" # Redis host address Redis 的地址，在 docker compose 网络中，直接引用服务名称 + # CRAWLAB_SERVER_LANG_NODE: "Y" # 预安装 Node.js 语言环境 ports: - "8080:8080" # frontend port mapping 前端端口映射 - "8000:8000" # backend port mapping 后端端口映射 depends_on: - mongo - redis + volumes: + - "/Users/marvzhang/projects/crawlab-team/crawlab/docker_init.sh:/app/docker_init.sh" worker: image: tikazyq/crawlab:latest container_name: worker diff --git a/docker_init.sh b/docker_init.sh index 97c505dc..648634cd 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # replace default api path to new one if [ "${CRAWLAB_API_ADDRESS}" = "" ]; @@ -22,5 +22,12 @@ fi # start nginx service nginx start +# install languages: Node.js +if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; +then + echo "installing node.js" + /bin/sh /app/backend/scripts/install-nodejs.sh +fi + # start backend crawlab \ No newline at end of file diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index a13b4c7a..1f29e89c 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -51,6 +51,10 @@ + + {{$t('Run')}} + - - {{$t('Run')}} - {{$t('Save')}} diff --git a/frontend/src/components/Node/NodeInstallation.vue b/frontend/src/components/Node/NodeInstallation.vue new file mode 100644 index 00000000..7cf4f6b9 --- /dev/null +++ b/frontend/src/components/Node/NodeInstallation.vue @@ -0,0 +1,304 @@ + + + + + diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index abd580c8..852e79be 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -65,6 +65,8 @@ export default { 'Back': '返回', 'New File': '新建文件', 'Rename': '重命名', + 'Install': '安装', + 'Uninstall': '卸载', // 主页 'Total Tasks': '总任务数', @@ -85,6 +87,8 @@ export default { 'Node Network': '节点拓扑图', 'Master': '主节点', 'Worker': '工作节点', + 'Installation': '安装', + 'Search Dependencies': '搜索依赖', // 节点列表 'IP': 'IP地址', @@ -262,6 +266,8 @@ export default { 'ARCH': '操作架构', 'Number of CPU': 'CPU数', 'Executables': '执行文件', + 'Latest Version': '最新版本', + 'Version': '版本', // 弹出框 'Notification': '提示', @@ -290,7 +296,23 @@ export default { 'NOTE: When uploading a zip file, please zip your spider files from the ROOT DIRECTORY.': '注意: 上传 zip 文件时，请从 根目录 下开始压缩爬虫文件。', 'English': 'English', 'Are you sure to delete the schedule task?': '确定删除定时任务?', + ' is not installed, do you want to install it?': ' 还没有安装，您是否打算安装它?', 'Disclaimer': '免责声明', + 'Please search dependencies': '请搜索依赖', + 'No Data': '暂无数据', + 'Show installed': '查看已安装', + 'Installing dependency successful': '安装依赖成功', + 'Installing dependency failed': '安装依赖失败', + 'You have successfully installed a dependency: ': '您已成功安装依赖: ', + 'The dependency installation is unsuccessful: ': '安装依赖失败: ', + 'Uninstalling dependency successful': '卸载依赖成功', + 'Uninstalling dependency failed': '卸载依赖失败', + 'You have successfully uninstalled a dependency: ': '您已成功卸载依赖: ', + 'The dependency uninstallation is unsuccessful: ': '卸载依赖失败: ', + 'Installing language successful': '安装语言成功', + 'Installing language failed': '安装语言失败', + 'You have successfully installed a language: ': '您已成功安装语言: ', + 'The language installation is unsuccessful: ': '安装语言失败: ', // 登录 'Sign in': '登录', diff --git a/frontend/src/store/modules/node.js b/frontend/src/store/modules/node.js index 5e21a222..873e53e8 100644 --- a/frontend/src/store/modules/node.js +++ b/frontend/src/store/modules/node.js @@ -3,7 +3,7 @@ import request from '../../api/request' const state = { // NodeList nodeList: [], - nodeForm: { _id: {} }, + nodeForm: {}, // spider to deploy/run activeSpider: {} diff --git a/frontend/src/views/node/NodeDetail.vue b/frontend/src/views/node/NodeDetail.vue index 527da581..10364b01 100644 --- a/frontend/src/views/node/NodeDetail.vue +++ b/frontend/src/views/node/NodeDetail.vue @@ -13,6 +13,9 @@ + + + {{$t('Deployed Spiders')}} @@ -25,11 +28,13 @@ import { mapState } from 'vuex' import NodeOverview from '../../components/Overview/NodeOverview' +import NodeInstallation from '../../components/Node/NodeInstallation' export default { name: 'NodeDetail', components: { - NodeOverview + NodeOverview, + NodeInstallation }, data () { return { @@ -43,7 +48,9 @@ export default { ]) }, methods: { - onTabClick () { + onTabClick (name) { + if (name === 'installation') { + } }, onNodeChange (id) { this.$router.push(`/nodes/${id}`) diff --git a/jenkins/master/docker-compose.yaml b/jenkins/master/docker-compose.yaml index ff9dd64e..ed649aa8 100644 --- a/jenkins/master/docker-compose.yaml +++ b/jenkins/master/docker-compose.yaml @@ -9,6 +9,7 @@ services: CRAWLAB_MONGO_HOST: "mongo" CRAWLAB_REDIS_ADDRESS: "redis" CRAWLAB_LOG_PATH: "/var/logs/crawlab" + CRAWLAB_SETTING_ALLOWREGISTER: "Y" ports: - "8080:8080" # frontend - "8000:8000" # backend diff --git a/wait-for-it.sh b/wait-for-it.sh deleted file mode 100755 index 607a7d67..00000000 --- a/wait-for-it.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available - -WAITFORIT_cmdname=${0##*/} - -echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - else - echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" - fi - WAITFORIT_start_ts=$(date +%s) - while : - do - if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then - nc -z $WAITFORIT_HOST $WAITFORIT_PORT - WAITFORIT_result=$? - else - (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 - WAITFORIT_result=$? - fi - if [[ $WAITFORIT_result -eq 0 ]]; then - WAITFORIT_end_ts=$(date +%s) - echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" - break - fi - sleep 1 - done - return $WAITFORIT_result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $WAITFORIT_QUIET -eq 1 ]]; then - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - else - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - fi - WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT - wait $WAITFORIT_PID - WAITFORIT_RESULT=$? - if [[ $WAITFORIT_RESULT -ne 0 ]]; then - echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - fi - return $WAITFORIT_RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - WAITFORIT_hostport=(${1//:/ }) - WAITFORIT_HOST=${WAITFORIT_hostport[0]} - WAITFORIT_PORT=${WAITFORIT_hostport[1]} - shift 1 - ;; - --child) - WAITFORIT_CHILD=1 - shift 1 - ;; - -q | --quiet) - WAITFORIT_QUIET=1 - shift 1 - ;; - -s | --strict) - WAITFORIT_STRICT=1 - shift 1 - ;; - -h) - WAITFORIT_HOST="$2" - if [[ $WAITFORIT_HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - WAITFORIT_HOST="${1#*=}" - shift 1 - ;; - -p) - WAITFORIT_PORT="$2" - if [[ $WAITFORIT_PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - WAITFORIT_PORT="${1#*=}" - shift 1 - ;; - -t) - WAITFORIT_TIMEOUT="$2" - if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - WAITFORIT_TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - WAITFORIT_CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} -WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} -WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} -WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} - -# check to see if timeout is from busybox? -WAITFORIT_TIMEOUT_PATH=$(type -p timeout) -WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) -if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then - WAITFORIT_ISBUSY=1 - WAITFORIT_BUSYTIMEFLAG="-t" - -else - WAITFORIT_ISBUSY=0 - WAITFORIT_BUSYTIMEFLAG="" -fi - -if [[ $WAITFORIT_CHILD -gt 0 ]]; then - wait_for - WAITFORIT_RESULT=$? - exit $WAITFORIT_RESULT -else - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - wait_for_wrapper - WAITFORIT_RESULT=$? - else - wait_for - WAITFORIT_RESULT=$? - fi -fi - -if [[ $WAITFORIT_CLI != "" ]]; then - if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then - echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" - exit $WAITFORIT_RESULT - fi - exec "${WAITFORIT_CLI[@]}" -else - exit $WAITFORIT_RESULT -fi \ No newline at end of file