diff --git a/.github/workflows/dockerpush.yml b/.github/workflows/dockerpush.yml index be968fe2..803059f4 100644 --- a/.github/workflows/dockerpush.yml +++ b/.github/workflows/dockerpush.yml @@ -64,3 +64,6 @@ jobs: if [ "$VERSION" == "develop" ]; then curl ${{ secrets.JENKINS_DEVELOP_URL }} fi + if [ "$VERSION" == "master" ]; then + curl ${{ secrets.JENKINS_DEMO_URL }} + fi diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index c28063d6..4099e43a 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,3 +1,21 @@ +# 0.4.8 (2020-03-11) +### 功能 / 优化 +- **支持更多编程语言安装**. 现在用户可以安装或预装更多的编程语言，包括 Java、.Net Core、PHP. +- **安装 UI 优化**. 用户能够更好的查看和管理节点列表页的安装. +- **更多 Git 支持**. 允许用户查看 Git Commits 记录，并 Checkout 到相应 Commit. +- **支持用 Hostname 作为节点注册类型**. 用户可以将 hostname 作为节点的唯一识别号. +- **RPC 支持**. 加入 RPC 支持来更好的管理节点通信. +- **是否在主节点运行开关**. 用户可以决定是否在主节点运行，如果为否，则所有任务将在工作节点上运行. +- **默认禁用教程**. +- **加入相关文档侧边栏**. +- **加载页面优化**. + +### Bug 修复 +- **重复节点**. [#391](https://github.com/crawlab-team/crawlab/issues/391) +- **重复上传爬虫**. [#603](https://github.com/crawlab-team/crawlab/issues/603) +- **节点第三方模块安装失败导致 节点安装第三方部分无法使用**. [#609](https://github.com/crawlab-team/crawlab/issues/609) +- **离线节点也会创建任务**. [#622](https://github.com/crawlab-team/crawlab/issues/622) + # 0.4.7 (2020-02-24) ### 功能 / 优化 - **更好的支持 Scrapy**. 爬虫识别，`settings.py` 配置，日志级别选择，爬虫选择. [#435](https://github.com/crawlab-team/crawlab/issues/435) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e1e1e52..0b4792f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# 0.4.8 (2020-03-11) +### Features / Enhancement +- **Support Installations of More Programming Languages**. Now users can install or pre-install more programming languages including Java, .Net Core and PHP. +- **Installation UI Optimization**. Users can better view and manage installations on Node List page. +- **More Git Support**. Allow users to view Git Commits record, and allow checkout to corresponding commit. +- **Support Hostname Node Registration Type**. Users can set hostname as the node key as the unique identifier. +- **RPC Support**. Added RPC support to better manage node communication. +- **Run On Master Switch**. Users can determine whether to run tasks on master. If not, all tasks will be run only on worker nodes. +- **Disabled Tutorial by Default**. +- **Added Related Documentation Sidebar**. +- **Loading Page Optimization**. + +### Bug Fixes +- **Duplicated Nodes**. [#391](https://github.com/crawlab-team/crawlab/issues/391) +- **Duplicated Spider Upload**. [#603](https://github.com/crawlab-team/crawlab/issues/603) +- **Failure in dependencies installation results in unusable dependency installation functionalities.**. [#609](https://github.com/crawlab-team/crawlab/issues/609) +- **Create Tasks for Offline Nodes**. [#622](https://github.com/crawlab-team/crawlab/issues/622) + # 0.4.7 (2020-02-24) ### Features / Enhancement - **Better Support for Scrapy**. Spiders identification, `settings.py` configuration, log level selection, spider selection. [#435](https://github.com/crawlab-team/crawlab/issues/435) diff --git a/Dockerfile b/Dockerfile index 48f41d5b..80a50c20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,8 @@ ENV DEBIAN_FRONTEND noninteractive ENV CRAWLAB_IS_DOCKER Y # install packages -RUN apt-get update \ +RUN chmod 777 /tmp \ + && apt-get update \ && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python diff --git a/Dockerfile.local b/Dockerfile.local index d49df7db..43cd874c 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -25,6 +25,9 @@ FROM ubuntu:latest # set as non-interactive ENV DEBIAN_FRONTEND noninteractive +# set CRAWLAB_IS_DOCKER +ENV CRAWLAB_IS_DOCKER Y + # install packages RUN chmod 777 /tmp \ && apt-get update \ diff --git a/README-zh.md b/README-zh.md index 4b662f23..0556532a 100644 --- a/README-zh.md +++ b/README-zh.md @@ -154,6 +154,10 @@ Docker部署的详情，请见[相关文档](https://tikazyq.github.io/crawlab-d ![](http://static-docs.crawlab.cn/schedule-v0.4.4.png) +#### 语言安装 + +![](http://static-docs.crawlab.cn/node-install-langs.png) + #### 依赖安装 ![](http://static-docs.crawlab.cn/node-install-dependencies.png) diff --git a/README.md b/README.md index 0ca2a772..8224cc77 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,10 @@ For Docker Deployment details, please refer to [relevant documentation](https:// ![](http://static-docs.crawlab.cn/schedule-v0.4.4.png) +#### Language Installation + +![](http://static-docs.crawlab.cn/node-install-langs.png) + #### Dependency Installation ![](http://static-docs.crawlab.cn/node-install-dependencies.png) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 1e98a2eb..6affb507 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -26,19 +26,24 @@ server: # mac地址/ip地址/hostname, 如果是ip，则需要手动指定IP type: "mac" ip: "" - lang: # 安装语言环境, Y 为安装，N 为不安装，只对 Docker 有效 + lang: # 安装语言环境, Y 为安装，N 为不安装 python: "Y" node: "N" + java: "N" + dotnet: "N" + php: "N" spider: path: "/app/spiders" task: workers: 4 other: tmppath: "/tmp" -version: 0.4.7 +version: 0.4.8 setting: allowRegister: "N" enableTutorial: "N" + runOnMaster: "Y" + demoSpiders: "N" notification: mail: server: '' diff --git a/backend/constants/rpc.go b/backend/constants/rpc.go index 6eebf0d5..d94eaf75 100644 --- a/backend/constants/rpc.go +++ b/backend/constants/rpc.go @@ -4,6 +4,6 @@ const ( RpcInstallLang = "install_lang" RpcInstallDep = "install_dep" RpcUninstallDep = "uninstall_dep" - RpcGetDepList = "get_dep_list" RpcGetInstalledDepList = "get_installed_dep_list" + RpcGetLang = "get_lang" ) diff --git a/backend/constants/system.go b/backend/constants/system.go index bec8b8c5..03491222 100644 --- a/backend/constants/system.go +++ b/backend/constants/system.go @@ -11,3 +11,10 @@ const ( Nodejs = "node" Java = "java" ) + +const ( + InstallStatusNotInstalled = "not-installed" + InstallStatusInstalling = "installing" + InstallStatusInstallingOther = "installing-other" + InstallStatusInstalled = "installed" +) diff --git a/backend/entity/node.go b/backend/entity/node.go index cf52fafb..ebe2047f 100644 --- a/backend/entity/node.go +++ b/backend/entity/node.go @@ -20,6 +20,9 @@ type NodeMessage struct { // 爬虫相关 SpiderId string `json:"spider_id"` //爬虫ID + // 语言相关 + Lang Lang `json:"lang"` + // 错误相关 Error string `json:"error"` } diff --git a/backend/entity/rpc.go b/backend/entity/rpc.go new file mode 100644 index 00000000..3f5ddcea --- /dev/null +++ b/backend/entity/rpc.go @@ -0,0 +1,11 @@ +package entity + +type RpcMessage struct { + Id string `json:"id"` + Method string `json:"method"` + NodeId string `json:"node_id"` + Params map[string]string `json:"params"` + Timeout int `json:"timeout"` + Result string `json:"result"` + Error string `json:"error"` +} diff --git a/backend/entity/system.go b/backend/entity/system.go index 4c15cc54..2738b55a 100644 --- a/backend/entity/system.go +++ b/backend/entity/system.go @@ -19,7 +19,9 @@ type Lang struct { ExecutableName string `json:"executable_name"` ExecutablePaths []string `json:"executable_paths"` DepExecutablePath string `json:"dep_executable_path"` - Installed bool `json:"installed"` + LockPath string `json:"lock_path"` + InstallScript string `json:"install_script"` + InstallStatus string `json:"install_status"` } type Dependency struct { diff --git a/backend/main.go b/backend/main.go index ea9c30df..d53991e2 100644 --- a/backend/main.go +++ b/backend/main.go @@ -9,6 +9,7 @@ import ( "crawlab/model" "crawlab/routes" "crawlab/services" + "crawlab/services/rpc" "github.com/apex/log" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" @@ -116,7 +117,7 @@ func main() { log.Info("initialized spider service successfully") // 初始化RPC服务 - if err := services.InitRpcService(); err != nil { + if err := rpc.InitRpcService(); err != nil { log.Error("init rpc service error:" + err.Error()) debug.PrintStack() panic(err) @@ -242,24 +243,26 @@ func main() { { authGroup.GET("/variables", routes.GetVariableList) // 列表 authGroup.PUT("/variable", routes.PutVariable) // 新增 - authGroup.POST("/variable/:id", routes.PostVariable) //修改 - authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除 + authGroup.POST("/variable/:id", routes.PostVariable) // 修改 + authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除 } // 项目 { authGroup.GET("/projects", routes.GetProjectList) // 列表 authGroup.GET("/projects/tags", routes.GetProjectTags) // 项目标签 - authGroup.PUT("/projects", routes.PutProject) //修改 + authGroup.PUT("/projects", routes.PutProject) // 修改 authGroup.POST("/projects/:id", routes.PostProject) // 新增 - authGroup.DELETE("/projects/:id", routes.DeleteProject) //删除 + authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除 } // 统计数据 authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 // 文件 authGroup.GET("/file", routes.GetFile) // 获取文件 // Git - authGroup.GET("/git/branches", routes.GetGitBranches) // 获取 Git 分支 + authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支 authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 + authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits + authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits } } diff --git a/backend/routes/git.go b/backend/routes/git.go index 56fbc683..23685b05 100644 --- a/backend/routes/git.go +++ b/backend/routes/git.go @@ -1,16 +1,19 @@ package routes import ( + "crawlab/model" "crawlab/services" "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" "net/http" ) -func GetGitBranches(c *gin.Context) { +func GetGitRemoteBranches(c *gin.Context) { url := c.Query("url") - branches, err := services.GetGitBranches(url) + branches, err := services.GetGitRemoteBranchesPlain(url) if err != nil { HandleError(http.StatusInternalServerError, c, err) + return } c.JSON(http.StatusOK, Response{ Status: "ok", @@ -27,3 +30,53 @@ func GetGitSshPublicKey(c *gin.Context) { Data: content, }) } + +func GetGitCommits(c *gin.Context) { + spiderId := c.Query("spider_id") + if spiderId == "" || !bson.IsObjectIdHex(spiderId) { + HandleErrorF(http.StatusInternalServerError, c, "invalid request") + return + } + spider, err := model.GetSpider(bson.ObjectIdHex(spiderId)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + commits, err := services.GetGitCommits(spider) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: commits, + }) +} + +func PostGitCheckout(c *gin.Context) { + type ReqBody struct { + SpiderId string `json:"spider_id"` + Hash string `json:"hash"` + } + var reqBody ReqBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + } + if reqBody.SpiderId == "" || !bson.IsObjectIdHex(reqBody.SpiderId) { + HandleErrorF(http.StatusInternalServerError, c, "invalid request") + return + } + spider, err := model.GetSpider(bson.ObjectIdHex(reqBody.SpiderId)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + if err := services.GitCheckout(spider, reqBody.Hash); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/routes/setting.go b/backend/routes/setting.go index 83976455..36bc46ca 100644 --- a/backend/routes/setting.go +++ b/backend/routes/setting.go @@ -7,8 +7,10 @@ import ( ) type SettingBody struct { - AllowRegister string `json:"allow_register"` - EnableTutorial string `json:"enable_tutorial"` + AllowRegister string `json:"allow_register"` + EnableTutorial string `json:"enable_tutorial"` + RunOnMaster string `json:"run_on_master"` + EnableDemoSpiders string `json:"enable_demo_spiders"` } func GetVersion(c *gin.Context) { @@ -23,8 +25,10 @@ func GetVersion(c *gin.Context) { func GetSetting(c *gin.Context) { body := SettingBody{ - AllowRegister: viper.GetString("setting.allowRegister"), - EnableTutorial: viper.GetString("setting.enableTutorial"), + AllowRegister: viper.GetString("setting.allowRegister"), + EnableTutorial: viper.GetString("setting.enableTutorial"), + RunOnMaster: viper.GetString("setting.runOnMaster"), + EnableDemoSpiders: viper.GetString("setting.enableDemoSpiders"), } c.JSON(http.StatusOK, Response{ diff --git a/backend/routes/system.go b/backend/routes/system.go index 8c443d2a..2caff7e4 100644 --- a/backend/routes/system.go +++ b/backend/routes/system.go @@ -4,6 +4,7 @@ import ( "crawlab/constants" "crawlab/entity" "crawlab/services" + "crawlab/services/rpc" "fmt" "github.com/gin-gonic/gin" "net/http" @@ -55,41 +56,20 @@ func GetInstalledDepList(c *gin.Context) { nodeId := c.Param("id") lang := c.Query("lang") var depList []entity.Dependency - if lang == constants.Python { - if services.IsMasterNode(nodeId) { - list, err := services.GetPythonLocalInstalledDepList(nodeId) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - depList = list - } else { - list, err := services.GetPythonRemoteInstalledDepList(nodeId) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - depList = list - } - } else if lang == constants.Nodejs { - if services.IsMasterNode(nodeId) { - list, err := services.GetNodejsLocalInstalledDepList(nodeId) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - depList = list - } else { - list, err := services.GetNodejsRemoteInstalledDepList(nodeId) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - depList = list + if services.IsMasterNode(nodeId) { + list, err := rpc.GetInstalledDepsLocal(lang) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return } + depList = list } else { - HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang)) - return + list, err := rpc.GetInstalledDepsRemote(nodeId, lang) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list } c.JSON(http.StatusOK, Response{ @@ -155,41 +135,18 @@ func InstallDep(c *gin.Context) { return } - if reqBody.Lang == constants.Python { - if services.IsMasterNode(nodeId) { - _, err := services.InstallPythonLocalDep(reqBody.DepName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - } else { - _, err := services.InstallPythonRemoteDep(nodeId, reqBody.DepName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - } - } else if reqBody.Lang == constants.Nodejs { - if services.IsMasterNode(nodeId) { - _, err := services.InstallNodejsLocalDep(reqBody.DepName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - } else { - _, err := services.InstallNodejsRemoteDep(nodeId, reqBody.DepName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } + if services.IsMasterNode(nodeId) { + if err := rpc.InstallDepLocal(reqBody.Lang, reqBody.DepName); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return } } else { - HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang)) - return + if err := rpc.InstallDepRemote(nodeId, reqBody.Lang, reqBody.DepName); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } } - // TODO: check if install is successful - c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", @@ -209,41 +166,18 @@ func UninstallDep(c *gin.Context) { HandleError(http.StatusBadRequest, c, err) } - if reqBody.Lang == constants.Python { - if services.IsMasterNode(nodeId) { - _, err := services.UninstallPythonLocalDep(reqBody.DepName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - } else { - _, err := services.UninstallPythonRemoteDep(nodeId, reqBody.DepName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - } - } else if reqBody.Lang == constants.Nodejs { - if services.IsMasterNode(nodeId) { - _, err := services.UninstallNodejsLocalDep(reqBody.DepName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - } else { - _, err := services.UninstallNodejsRemoteDep(nodeId, reqBody.DepName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } + if services.IsMasterNode(nodeId) { + if err := rpc.UninstallDepLocal(reqBody.Lang, reqBody.DepName); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return } } else { - HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang)) - return + if err := rpc.UninstallDepRemote(nodeId, reqBody.Lang, reqBody.DepName); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } } - // TODO: check if uninstall is successful - c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", @@ -288,23 +222,18 @@ func InstallLang(c *gin.Context) { return } - if reqBody.Lang == constants.Nodejs { - if services.IsMasterNode(nodeId) { - _, err := services.InstallNodejsLocalLang() - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - } else { - _, err := services.InstallNodejsRemoteLang(nodeId) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } + if services.IsMasterNode(nodeId) { + _, err := rpc.InstallLangLocal(reqBody.Lang) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return } } else { - HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", reqBody.Lang)) - return + _, err := rpc.InstallLangRemote(nodeId, reqBody.Lang) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } } // TODO: check if install is successful diff --git a/backend/scripts/install-dotnet.sh b/backend/scripts/install-dotnet.sh new file mode 100755 index 00000000..ae2464c0 --- /dev/null +++ b/backend/scripts/install-dotnet.sh @@ -0,0 +1,17 @@ +# lock global +touch /tmp/install.lock + +# lock +touch /tmp/install-dotnet.lock + +wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb +dpkg -i packages-microsoft-prod.deb +apt-get install -y apt-transport-https +apt-get update +apt-get install -y dotnet-sdk-2.1 dotnet-runtime-2.1 aspnetcore-runtime-2.1 + +# unlock global +rm /tmp/install.lock + +# unlock +rm /tmp/install-dotnet.lock diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh index d99b16be..7cb8972c 100755 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -1,11 +1,19 @@ -#!/bin/env bash +#!/bin/bash + +# lock global +touch /tmp/install.lock # lock touch /tmp/install-java.lock # install java -apt-get update && apt-get install -y default-jdk --fix-missing +apt-get clean && \ + apt-get update --fix-missing && \ + apt-get install -y --fix-missing default-jdk ln -s /usr/bin/java /usr/local/bin/java # unlock rm /tmp/install-java.lock + +# unlock global +rm /tmp/install.lock diff --git a/backend/scripts/install-nodejs.sh b/backend/scripts/install-nodejs.sh index eada900a..cf01c7f8 100644 --- a/backend/scripts/install-nodejs.sh +++ b/backend/scripts/install-nodejs.sh @@ -1,4 +1,7 @@ -#!/bin/env bash +#!/bin/bash + +# lock global +touch /tmp/install.lock # lock touch /tmp/install-nodejs.lock @@ -39,3 +42,6 @@ npm install puppeteer-chromium-resolver crawlab-sdk -g --unsafe-perm=true --regi # unlock rm /tmp/install-nodejs.lock + +# unlock global +rm /tmp/install.lock diff --git a/backend/scripts/install-php.sh b/backend/scripts/install-php.sh new file mode 100755 index 00000000..bc6e3ad6 --- /dev/null +++ b/backend/scripts/install-php.sh @@ -0,0 +1,13 @@ +# lock global +touch /tmp/install.lock + +# lock +touch /tmp/install-php.lock + +apt-get install -y php + +# unlock global +rm /tmp/install.lock + +# unlock +rm /tmp/install-php.lock diff --git a/backend/scripts/install.sh b/backend/scripts/install.sh new file mode 100644 index 00000000..fe89d1b2 --- /dev/null +++ b/backend/scripts/install.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# install node.js +if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; +then + echo "installing node.js" + /bin/sh /app/backend/scripts/install-nodejs.sh + echo "installed node.js" +fi + +# install java +if [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ]; +then + echo "installing java" + /bin/sh /app/backend/scripts/install-java.sh + echo "installed java" +fi + +# install dotnet +if [ "${CRAWLAB_SERVER_LANG_DOTNET}" = "Y" ]; +then + echo "installing dotnet" + /bin/sh /app/backend/scripts/install-dotnet.sh + echo "installed dotnet" +fi diff --git a/backend/services/git.go b/backend/services/git.go index 1bb8b3b9..18d679de 100644 --- a/backend/services/git.go +++ b/backend/services/git.go @@ -12,6 +12,7 @@ import ( "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" "io/ioutil" "net/url" @@ -21,6 +22,7 @@ import ( "regexp" "runtime/debug" "strings" + "time" ) var GitCron *GitCronScheduler @@ -29,6 +31,102 @@ type GitCronScheduler struct { cron *cron.Cron } +type GitBranch struct { + Hash string `json:"hash"` + Name string `json:"name"` + Label string `json:"label"` +} + +type GitTag struct { + Hash string `json:"hash"` + Name string `json:"name"` + Label string `json:"label"` +} + +type GitCommit struct { + Hash string `json:"hash"` + TreeHash string `json:"tree_hash"` + Author string `json:"author"` + Email string `json:"email"` + Message string `json:"message"` + IsHead bool `json:"is_head"` + Ts time.Time `json:"ts"` + Branches []GitBranch `json:"branches"` + RemoteBranches []GitBranch `json:"remote_branches"` + Tags []GitTag `json:"tags"` +} + +func (g *GitCronScheduler) Start() error { + c := cron.New(cron.WithSeconds()) + + // 启动cron服务 + g.cron.Start() + + // 更新任务列表 + if err := g.Update(); err != nil { + log.Errorf("update scheduler error: %s", err.Error()) + debug.PrintStack() + return err + } + + // 每30秒更新一次任务列表 + spec := "*/30 * * * * *" + if _, err := c.AddFunc(spec, UpdateGitCron); err != nil { + log.Errorf("add func update schedulers error: %s", err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +func (g *GitCronScheduler) RemoveAll() { + entries := g.cron.Entries() + for i := 0; i < len(entries); i++ { + g.cron.Remove(entries[i].ID) + } +} + +func (g *GitCronScheduler) Update() error { + // 删除所有定时任务 + g.RemoveAll() + + // 获取开启 Git 自动同步的爬虫 + spiders, err := model.GetSpiderAllList(bson.M{"git_auto_sync": true}) + if err != nil { + log.Errorf("get spider list error: %s", err.Error()) + debug.PrintStack() + return err + } + + // 遍历任务列表 + for _, s := range spiders { + // 添加到定时任务 + if err := g.AddJob(s); err != nil { + log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), s.Name, s.GitSyncFrequency) + debug.PrintStack() + return err + } + } + + return nil +} + +func (g *GitCronScheduler) AddJob(s model.Spider) error { + spec := s.GitSyncFrequency + + // 添加定时任务 + _, err := g.cron.AddFunc(spec, AddGitCronJob(s)) + if err != nil { + log.Errorf("add func task error: %s", err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +// 保存爬虫Git同步错误 func SaveSpiderGitSyncError(s model.Spider, errMsg string) { s, _ = model.GetSpider(s.Id) s.GitSyncError = errMsg @@ -39,7 +137,8 @@ func SaveSpiderGitSyncError(s model.Spider, errMsg string) { } } -func GetGitBranches(url string) (branches []string, err error) { +// 获得Git分支 +func GetGitRemoteBranchesPlain(url string) (branches []string, err error) { var stdout bytes.Buffer var stderr bytes.Buffer @@ -63,6 +162,7 @@ func GetGitBranches(url string) (branches []string, err error) { return branches, nil } +// 重置爬虫Git func ResetSpiderGit(s model.Spider) (err error) { // 删除文件夹 if err := os.RemoveAll(s.Src); err != nil { @@ -86,6 +186,7 @@ func ResetSpiderGit(s model.Spider) (err error) { return nil } +// 同步爬虫Git func SyncSpiderGit(s model.Spider) (err error) { // 如果 .git 不存在，初始化一个仓库 if !utils.Exists(path.Join(s.Src, ".git")) { @@ -165,6 +266,7 @@ func SyncSpiderGit(s model.Spider) (err error) { RemoteName: "origin", Force: true, Auth: auth, + Tags: git.AllTags, }) // 获得 WorkTree @@ -178,8 +280,10 @@ func SyncSpiderGit(s model.Spider) (err error) { // 拉取 repo if err := wt.Pull(&git.PullOptions{ - RemoteName: "origin", - Auth: auth, + RemoteName: "origin", + Auth: auth, + ReferenceName: plumbing.HEAD, + SingleBranch: false, }); err != nil { if err.Error() == "already up-to-date" { // 检查是否为 Scrapy @@ -221,76 +325,7 @@ func SyncSpiderGit(s model.Spider) (err error) { return nil } -func (g *GitCronScheduler) Start() error { - c := cron.New(cron.WithSeconds()) - - // 启动cron服务 - g.cron.Start() - - // 更新任务列表 - if err := g.Update(); err != nil { - log.Errorf("update scheduler error: %s", err.Error()) - debug.PrintStack() - return err - } - - // 每30秒更新一次任务列表 - spec := "*/30 * * * * *" - if _, err := c.AddFunc(spec, UpdateGitCron); err != nil { - log.Errorf("add func update schedulers error: %s", err.Error()) - debug.PrintStack() - return err - } - - return nil -} - -func (g *GitCronScheduler) RemoveAll() { - entries := g.cron.Entries() - for i := 0; i < len(entries); i++ { - g.cron.Remove(entries[i].ID) - } -} - -func (g *GitCronScheduler) Update() error { - // 删除所有定时任务 - g.RemoveAll() - - // 获取开启 Git 自动同步的爬虫 - spiders, err := model.GetSpiderAllList(bson.M{"git_auto_sync": true}) - if err != nil { - log.Errorf("get spider list error: %s", err.Error()) - debug.PrintStack() - return err - } - - // 遍历任务列表 - for _, s := range spiders { - // 添加到定时任务 - if err := g.AddJob(s); err != nil { - log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), s.Name, s.GitSyncFrequency) - debug.PrintStack() - return err - } - } - - return nil -} - -func (g *GitCronScheduler) AddJob(s model.Spider) error { - spec := s.GitSyncFrequency - - // 添加定时任务 - _, err := g.cron.AddFunc(spec, AddGitCronJob(s)) - if err != nil { - log.Errorf("add func task error: %s", err.Error()) - debug.PrintStack() - return err - } - - return nil -} - +// 添加Git定时任务 func AddGitCronJob(s model.Spider) func() { return func() { if err := SyncSpiderGit(s); err != nil { @@ -301,6 +336,7 @@ func AddGitCronJob(s model.Spider) func() { } } +// 更新Git定时任务 func UpdateGitCron() { if err := GitCron.Update(); err != nil { log.Errorf(err.Error()) @@ -308,6 +344,7 @@ func UpdateGitCron() { } } +// 获取SSH公钥 func GetGitSshPublicKey() string { if !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh")) || !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa")) || @@ -322,3 +359,198 @@ func GetGitSshPublicKey() string { } return string(content) } + +// 获取Git分支 +func GetGitBranches(s model.Spider) (branches []GitBranch, err error) { + // 打开 repo + repo, err := git.PlainOpen(s.Src) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return branches, err + } + + iter, err := repo.Branches() + if iter == nil { + return branches, nil + } + if err := iter.ForEach(func(reference *plumbing.Reference) error { + branches = append(branches, GitBranch{ + Hash: reference.Hash().String(), + Name: reference.Name().String(), + Label: reference.Name().Short(), + }) + return nil + }); err != nil { + return branches, err + } + + return branches, nil +} + +// 获取Git Tags +func GetGitTags(s model.Spider) (tags []GitTag, err error) { + // 打开 repo + repo, err := git.PlainOpen(s.Src) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return tags, err + } + + iter, err := repo.Tags() + if iter == nil { + return tags, nil + } + if err := iter.ForEach(func(reference *plumbing.Reference) error { + tags = append(tags, GitTag{ + Hash: reference.Hash().String(), + Name: reference.Name().String(), + Label: reference.Name().Short(), + }) + return nil + }); err != nil { + return tags, err + } + + return tags, nil +} + +// 获取Git Head Hash +func GetGitHeadHash(repo *git.Repository) string { + head, _ := repo.Head() + return head.Hash().String() +} + +// 获取Git远端分支 +func GetGitRemoteBranches(s model.Spider) (branches []GitBranch, err error) { + // 打开 repo + repo, err := git.PlainOpen(s.Src) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return branches, err + } + + iter, err := repo.References() + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return branches, err + } + if err := iter.ForEach(func(reference *plumbing.Reference) error { + if reference.Name().IsRemote() { + log.Infof(reference.Hash().String()) + log.Infof(reference.Name().String()) + branches = append(branches, GitBranch{ + Hash: reference.Hash().String(), + Name: reference.Name().String(), + Label: reference.Name().Short(), + }) + } + return nil + }); err != nil { + log.Error(err.Error()) + debug.PrintStack() + return branches, err + } + return branches, err +} + +// 获取Git Commits +func GetGitCommits(s model.Spider) (commits []GitCommit, err error) { + // 打开 repo + repo, err := git.PlainOpen(s.Src) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return commits, err + } + + // 获取分支列表 + branches, err := GetGitBranches(s) + branchesDict := map[string][]GitBranch{} + for _, b := range branches { + branchesDict[b.Hash] = append(branchesDict[b.Hash], b) + } + + // 获取分支列表 + remoteBranches, err := GetGitRemoteBranches(s) + remoteBranchesDict := map[string][]GitBranch{} + for _, b := range remoteBranches { + remoteBranchesDict[b.Hash] = append(remoteBranchesDict[b.Hash], b) + } + + // 获取标签列表 + tags, err := GetGitTags(s) + tagsDict := map[string][]GitTag{} + for _, t := range tags { + tagsDict[t.Hash] = append(tagsDict[t.Hash], t) + } + + // 获取日志遍历器 + iter, err := repo.Log(&git.LogOptions{ + All: true, + }) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return commits, err + } + + // 遍历日志 + if err := iter.ForEach(func(commit *object.Commit) error { + gc := GitCommit{ + Hash: commit.Hash.String(), + TreeHash: commit.TreeHash.String(), + Message: commit.Message, + Author: commit.Author.Name, + Email: commit.Author.Email, + Ts: commit.Author.When, + IsHead: commit.Hash.String() == GetGitHeadHash(repo), + Branches: branchesDict[commit.Hash.String()], + RemoteBranches: remoteBranchesDict[commit.Hash.String()], + Tags: tagsDict[commit.Hash.String()], + } + commits = append(commits, gc) + return nil + }); err != nil { + log.Error(err.Error()) + debug.PrintStack() + return commits, err + } + + return commits, nil +} + +func GitCheckout(s model.Spider, hash string) (err error) { + // 打开 repo + repo, err := git.PlainOpen(s.Src) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return err + } + + // 获取worktree + wt, err := repo.Worktree() + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return err + } + + // Checkout + if err := wt.Checkout(&git.CheckoutOptions{ + Hash: plumbing.NewHash(hash), + Create: false, + Force: true, + Keep: false, + }); err != nil { + log.Error(err.Error()) + debug.PrintStack() + return err + } + + return nil +} diff --git a/backend/services/msg_handler/handler.go b/backend/services/msg_handler/handler.go index b8b8e231..bd98fce6 100644 --- a/backend/services/msg_handler/handler.go +++ b/backend/services/msg_handler/handler.go @@ -11,7 +11,7 @@ type Handler interface { } func GetMsgHandler(msg entity.NodeMessage) Handler { - log.Infof("received msg , type is : %s", msg.Type) + log.Debugf("received msg , type is : %s", msg.Type) if msg.Type == constants.MsgTypeGetLog || msg.Type == constants.MsgTypeRemoveLog { // 日志相关 return &Log{ diff --git a/backend/services/node.go b/backend/services/node.go index f8db160a..89187a81 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -15,6 +15,7 @@ import ( "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "github.com/gomodule/redigo/redis" + "github.com/spf13/viper" "runtime/debug" "time" ) @@ -104,6 +105,19 @@ func UpdateNodeStatus() { model.ResetNodeStatusToOffline(list) } +func getNodeName(data *Data) string { + registerType := viper.GetString("server.register.type") + if registerType == constants.RegisterTypeMac { + return data.Ip + } else if registerType == constants.RegisterTypeIp { + return data.Ip + } else if registerType == constants.RegisterTypeHostname { + return data.Hostname + } else { + return data.Ip + } +} + // 处理节点信息 func handleNodeInfo(key string, data *Data) { // 添加同步锁 @@ -122,7 +136,7 @@ func handleNodeInfo(key string, data *Data) { // 数据库不存在该节点 node = model.Node{ Key: key, - Name: data.Ip, + Name: getNodeName(data), Ip: data.Ip, Port: "8000", Mac: data.Mac, diff --git a/backend/services/register/register.go b/backend/services/register/register.go index 8169ae65..98e70707 100644 --- a/backend/services/register/register.go +++ b/backend/services/register/register.go @@ -10,6 +10,7 @@ import ( "os/exec" "reflect" "runtime/debug" + "strings" "sync" ) @@ -145,7 +146,7 @@ func getHostname() (string, error) { return "", err } - return stdout.String(), nil + return strings.Replace(stdout.String(), "\n", "", -1), nil } // ===================== 获得注册简单工厂 ===================== @@ -156,11 +157,6 @@ var once sync.Once func GetRegister() Register { once.Do(func() { - - if register != nil { - register = register - } - registerType := viper.GetString("server.register.type") if registerType == constants.RegisterTypeMac { register = &MacRegister{} diff --git a/backend/services/rpc.go b/backend/services/rpc.go deleted file mode 100644 index 66c04369..00000000 --- a/backend/services/rpc.go +++ /dev/null @@ -1,234 +0,0 @@ -package services - -import ( - "crawlab/constants" - "crawlab/database" - "crawlab/entity" - "crawlab/model" - "crawlab/utils" - "encoding/json" - "fmt" - "github.com/apex/log" - "github.com/gomodule/redigo/redis" - uuid "github.com/satori/go.uuid" - "runtime/debug" -) - -type RpcMessage struct { - Id string `json:"id"` - Method string `json:"method"` - Params map[string]string `json:"params"` - Result string `json:"result"` -} - -func RpcServerInstallLang(msg RpcMessage) RpcMessage { - lang := GetRpcParam("lang", msg.Params) - if lang == constants.Nodejs { - output, _ := InstallNodejsLocalLang() - msg.Result = output - } - return msg -} - -func RpcClientInstallLang(nodeId string, lang string) (output string, err error) { - params := map[string]string{} - params["lang"] = lang - - data, err := RpcClientFunc(nodeId, constants.RpcInstallLang, params, 600)() - if err != nil { - return - } - - output = data - - return -} - -func RpcServerInstallDep(msg RpcMessage) RpcMessage { - lang := GetRpcParam("lang", msg.Params) - depName := GetRpcParam("dep_name", msg.Params) - if lang == constants.Python { - output, _ := InstallPythonLocalDep(depName) - msg.Result = output - } - return msg -} - -func RpcClientInstallDep(nodeId string, lang string, depName string) (output string, err error) { - params := map[string]string{} - params["lang"] = lang - params["dep_name"] = depName - - data, err := RpcClientFunc(nodeId, constants.RpcInstallDep, params, 10)() - if err != nil { - return - } - - output = data - - return -} - -func RpcServerUninstallDep(msg RpcMessage) RpcMessage { - lang := GetRpcParam("lang", msg.Params) - depName := GetRpcParam("dep_name", msg.Params) - if lang == constants.Python { - output, _ := UninstallPythonLocalDep(depName) - msg.Result = output - } - return msg -} - -func RpcClientUninstallDep(nodeId string, lang string, depName string) (output string, err error) { - params := map[string]string{} - params["lang"] = lang - params["dep_name"] = depName - - data, err := RpcClientFunc(nodeId, constants.RpcUninstallDep, params, 60)() - if err != nil { - return - } - - output = data - - return -} - -func RpcServerGetInstalledDepList(nodeId string, msg RpcMessage) RpcMessage { - lang := GetRpcParam("lang", msg.Params) - if lang == constants.Python { - depList, _ := GetPythonLocalInstalledDepList(nodeId) - resultStr, _ := json.Marshal(depList) - msg.Result = string(resultStr) - } else if lang == constants.Nodejs { - depList, _ := GetNodejsLocalInstalledDepList(nodeId) - resultStr, _ := json.Marshal(depList) - msg.Result = string(resultStr) - } - return msg -} - -func RpcClientGetInstalledDepList(nodeId string, lang string) (list []entity.Dependency, err error) { - params := map[string]string{} - params["lang"] = lang - - data, err := RpcClientFunc(nodeId, constants.RpcGetInstalledDepList, params, 10)() - if err != nil { - return - } - - // 反序列化结果 - if err := json.Unmarshal([]byte(data), &list); err != nil { - return list, err - } - - return -} - -func RpcClientFunc(nodeId string, method string, params map[string]string, timeout int) func() (string, error) { - return func() (result string, err error) { - // 请求ID - id := uuid.NewV4().String() - - // 构造RPC消息 - msg := RpcMessage{ - Id: id, - Method: method, - Params: params, - Result: "", - } - - // 发送RPC消息 - msgStr := ObjectToString(msg) - if err := database.RedisClient.LPush(fmt.Sprintf("rpc:%s", nodeId), msgStr); err != nil { - return result, err - } - - // 获取RPC回复消息 - dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", nodeId), timeout) - if err != nil { - return result, err - } - - // 反序列化消息 - if err := json.Unmarshal([]byte(dataStr), &msg); err != nil { - return result, err - } - - return msg.Result, err - } -} - -func GetRpcParam(key string, params map[string]string) string { - return params[key] -} - -func ObjectToString(params interface{}) string { - bytes, _ := json.Marshal(params) - return utils.BytesToString(bytes) -} - -var IsRpcStopped = false - -func StopRpcService() { - IsRpcStopped = true -} - -func InitRpcService() error { - go func() { - for { - // 获取当前节点 - node, err := model.GetCurrentNode() - if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - continue - } - - // 获取获取消息队列信息 - dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 0) - if err != nil { - if err != redis.ErrNil { - log.Errorf(err.Error()) - debug.PrintStack() - } - continue - } - - // 反序列化消息 - var msg RpcMessage - if err := json.Unmarshal([]byte(dataStr), &msg); err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - continue - } - - // 根据Method调用本地方法 - var replyMsg RpcMessage - if msg.Method == constants.RpcInstallDep { - replyMsg = RpcServerInstallDep(msg) - } else if msg.Method == constants.RpcUninstallDep { - replyMsg = RpcServerUninstallDep(msg) - } else if msg.Method == constants.RpcInstallLang { - replyMsg = RpcServerInstallLang(msg) - } else if msg.Method == constants.RpcGetInstalledDepList { - replyMsg = RpcServerGetInstalledDepList(node.Id.Hex(), msg) - } else { - continue - } - - // 发送返回消息 - if err := database.RedisClient.LPush(fmt.Sprintf("rpc:%s", node.Id.Hex()), ObjectToString(replyMsg)); err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - continue - } - - // 如果停止RPC服务，则返回 - if IsRpcStopped { - return - } - } - }() - return nil -} diff --git a/backend/services/rpc/base.go b/backend/services/rpc/base.go new file mode 100644 index 00000000..ea4c7a62 --- /dev/null +++ b/backend/services/rpc/base.go @@ -0,0 +1,132 @@ +package rpc + +import ( + "crawlab/constants" + "crawlab/database" + "crawlab/entity" + "crawlab/model" + "crawlab/utils" + "encoding/json" + "errors" + "fmt" + "github.com/apex/log" + "github.com/gomodule/redigo/redis" + uuid "github.com/satori/go.uuid" + "runtime/debug" +) + +// RPC服务基础类 +type Service interface { + ServerHandle() (entity.RpcMessage, error) + ClientHandle() (interface{}, error) +} + +// 客户端处理消息函数 +func ClientFunc(msg entity.RpcMessage) func() (entity.RpcMessage, error) { + return func() (replyMsg entity.RpcMessage, err error) { + // 请求ID + msg.Id = uuid.NewV4().String() + + // 发送RPC消息 + msgStr := utils.ObjectToString(msg) + if err := database.RedisClient.LPush(fmt.Sprintf("rpc:%s", msg.NodeId), msgStr); err != nil { + log.Errorf("RpcClientFunc error: " + err.Error()) + debug.PrintStack() + return replyMsg, err + } + + // 获取RPC回复消息 + dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s:%s", msg.NodeId, msg.Id), msg.Timeout) + if err != nil { + log.Errorf("RpcClientFunc error: " + err.Error()) + debug.PrintStack() + return replyMsg, err + } + + // 反序列化消息 + if err := json.Unmarshal([]byte(dataStr), &replyMsg); err != nil { + log.Errorf("RpcClientFunc error: " + err.Error()) + debug.PrintStack() + return replyMsg, err + } + + // 如果返回消息有错误，返回错误 + if replyMsg.Error != "" { + return replyMsg, errors.New(replyMsg.Error) + } + + return + } +} + +// 获取RPC服务 +func GetService(msg entity.RpcMessage) Service { + switch msg.Method { + case constants.RpcInstallLang: + return &InstallLangService{msg: msg} + case constants.RpcInstallDep: + return &InstallDepService{msg: msg} + case constants.RpcUninstallDep: + return &UninstallDepService{msg: msg} + case constants.RpcGetLang: + return &GetLangService{msg: msg} + case constants.RpcGetInstalledDepList: + return &GetInstalledDepsService{msg: msg} + } + return nil +} + +// 处理RPC消息 +func handleMsg(msgStr string, node model.Node) { + // 反序列化消息 + var msg entity.RpcMessage + if err := json.Unmarshal([]byte(msgStr), &msg); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + } + + // 获取service + service := GetService(msg) + + // 根据Method调用本地方法 + replyMsg, err := service.ServerHandle() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + } + + // 发送返回消息 + if err := database.RedisClient.LPush(fmt.Sprintf("rpc:%s:%s", node.Id.Hex(), replyMsg.Id), utils.ObjectToString(replyMsg)); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + } +} + +// 初始化服务端RPC服务 +func InitRpcService() error { + go func() { + for { + // 获取当前节点 + node, err := model.GetCurrentNode() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + + // 获取获取消息队列信息 + msgStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 0) + if err != nil { + if err != redis.ErrNil { + log.Errorf(err.Error()) + debug.PrintStack() + } + continue + } + + // 处理消息 + go handleMsg(msgStr, node) + } + }() + return nil +} diff --git a/backend/services/rpc/get_installed_deps.go b/backend/services/rpc/get_installed_deps.go new file mode 100644 index 00000000..9017fa64 --- /dev/null +++ b/backend/services/rpc/get_installed_deps.go @@ -0,0 +1,123 @@ +package rpc + +import ( + "crawlab/constants" + "crawlab/entity" + "crawlab/utils" + "encoding/json" + "os/exec" + "regexp" + "runtime/debug" + "strings" +) + +type GetInstalledDepsService struct { + msg entity.RpcMessage +} + +func (s *GetInstalledDepsService) ServerHandle() (entity.RpcMessage, error) { + lang := utils.GetRpcParam("lang", s.msg.Params) + deps, err := GetInstalledDepsLocal(lang) + if err != nil { + s.msg.Error = err.Error() + return s.msg, err + } + resultStr, _ := json.Marshal(deps) + s.msg.Result = string(resultStr) + return s.msg, nil +} + +func (s *GetInstalledDepsService) ClientHandle() (o interface{}, err error) { + // 发起 RPC 请求，获取服务端数据 + s.msg, err = ClientFunc(s.msg)() + if err != nil { + return o, err + } + + // 反序列化 + var output []entity.Dependency + if err := json.Unmarshal([]byte(s.msg.Result), &output); err != nil { + return o, err + } + o = output + + return +} + +// 获取本地已安装依赖列表 +func GetInstalledDepsLocal(lang string) (deps []entity.Dependency, err error) { + if lang == constants.Python { + deps, err = GetPythonInstalledDepListLocal() + } else if lang == constants.Nodejs { + deps, err = GetNodejsInstalledDepListLocal() + } + return deps, err +} + +// 获取Python本地已安装依赖列表 +func GetPythonInstalledDepListLocal() ([]entity.Dependency, error) { + var list []entity.Dependency + + cmd := exec.Command("pip", "freeze") + outputBytes, err := cmd.Output() + if err != nil { + debug.PrintStack() + return list, err + } + + for _, line := range strings.Split(string(outputBytes), "\n") { + arr := strings.Split(line, "==") + if len(arr) < 2 { + continue + } + dep := entity.Dependency{ + Name: strings.ToLower(arr[0]), + Version: arr[1], + Installed: true, + } + list = append(list, dep) + } + + return list, nil +} + +// 获取Node.js本地已安装依赖列表 +func GetNodejsInstalledDepListLocal() ([]entity.Dependency, error) { + var list []entity.Dependency + + cmd := exec.Command("npm", "ls", "-g", "--depth", "0") + outputBytes, _ := cmd.Output() + + regex := regexp.MustCompile("\\s(.*)@(.*)") + for _, line := range strings.Split(string(outputBytes), "\n") { + arr := regex.FindStringSubmatch(line) + if len(arr) < 3 { + continue + } + dep := entity.Dependency{ + Name: strings.ToLower(arr[1]), + Version: arr[2], + Installed: true, + } + list = append(list, dep) + } + + return list, nil +} + +func GetInstalledDepsRemote(nodeId string, lang string) (deps []entity.Dependency, err error) { + params := make(map[string]string) + params["lang"] = lang + s := GetService(entity.RpcMessage{ + NodeId: nodeId, + Method: constants.RpcGetInstalledDepList, + Params: params, + Timeout: 60, + }) + o, err := s.ClientHandle() + if err != nil { + return + } + deps = o.([]entity.Dependency) + return +} diff --git a/backend/services/rpc/get_lang.go b/backend/services/rpc/get_lang.go new file mode 100644 index 00000000..d0662984 --- /dev/null +++ b/backend/services/rpc/get_lang.go @@ -0,0 +1,82 @@ +package rpc + +import ( + "crawlab/constants" + "crawlab/entity" + "crawlab/utils" + "encoding/json" +) + +type GetLangService struct { + msg entity.RpcMessage +} + +func (s *GetLangService) ServerHandle() (entity.RpcMessage, error) { + langName := utils.GetRpcParam("lang", s.msg.Params) + lang := utils.GetLangFromLangNamePlain(langName) + l := GetLangLocal(lang) + lang.InstallStatus = l.InstallStatus + + // 序列化 + resultStr, _ := json.Marshal(lang) + s.msg.Result = string(resultStr) + return s.msg, nil +} + +func (s *GetLangService) ClientHandle() (o interface{}, err error) { + // 发起 RPC 请求，获取服务端数据 + s.msg, err = ClientFunc(s.msg)() + if err != nil { + return o, err + } + + var output entity.Lang + if err := json.Unmarshal([]byte(s.msg.Result), &output); err != nil { + return o, err + } + o = output + + return +} + +func GetLangLocal(lang entity.Lang) entity.Lang { + // 检查是否存在执行路径 + for _, p := range lang.ExecutablePaths { + if utils.Exists(p) { + lang.InstallStatus = constants.InstallStatusInstalled + return lang + } + } + + // 检查是否正在安装 + if utils.Exists(lang.LockPath) { + lang.InstallStatus = constants.InstallStatusInstalling + return lang + } + + // 检查其他语言是否在安装 + if utils.Exists("/tmp/install.lock") { + lang.InstallStatus = constants.InstallStatusInstallingOther + return lang + } + + lang.InstallStatus = constants.InstallStatusNotInstalled + return lang +} + +func GetLangRemote(nodeId string, lang entity.Lang) (l entity.Lang, err error) { + params := make(map[string]string) + params["lang"] = lang.ExecutableName + s := GetService(entity.RpcMessage{ + NodeId: nodeId, + Method: constants.RpcGetLang, + Params: params, + Timeout: 60, + }) + o, err := s.ClientHandle() + if err != nil { + return + } + l = o.(entity.Lang) + return +} diff --git a/backend/services/rpc/install_dep.go b/backend/services/rpc/install_dep.go new file mode 100644 index 00000000..e615688a --- /dev/null +++ b/backend/services/rpc/install_dep.go @@ -0,0 +1,100 @@ +package rpc + +import ( + "crawlab/constants" + "crawlab/entity" + "crawlab/utils" + "errors" + "fmt" + "github.com/apex/log" + "os/exec" + "runtime/debug" +) + +type InstallDepService struct { + msg entity.RpcMessage +} + +func (s *InstallDepService) ServerHandle() (entity.RpcMessage, error) { + lang := utils.GetRpcParam("lang", s.msg.Params) + depName := utils.GetRpcParam("dep_name", s.msg.Params) + if err := InstallDepLocal(lang, depName); err != nil { + s.msg.Error = err.Error() + return s.msg, err + } + s.msg.Result = "success" + return s.msg, nil +} + +func (s *InstallDepService) ClientHandle() (o interface{}, err error) { + // 发起 RPC 请求，获取服务端数据 + _, err = ClientFunc(s.msg)() + if err != nil { + return + } + + return +} + +func InstallDepLocal(lang string, depName string) error { + if lang == constants.Python { + _, err := InstallPythonDepLocal(depName) + if err != nil { + return err + } + } else if lang == constants.Nodejs { + _, err := InstallNodejsDepLocal(depName) + if err != nil { + return err + } + } else { + return errors.New(fmt.Sprintf("%s is not implemented", lang)) + } + return nil +} + +// 安装Python本地依赖 +func InstallPythonDepLocal(depName string) (string, error) { + // 依赖镜像URL + url := "https://pypi.tuna.tsinghua.edu.cn/simple" + + cmd := exec.Command("pip", "install", depName, "-i", url) + outputBytes, err := cmd.Output() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return fmt.Sprintf("error: %s", err.Error()), err + } + return string(outputBytes), nil +} + +func InstallNodejsDepLocal(depName string) (string, error) { + // 依赖镜像URL + url := "https://registry.npm.taobao.org" + + cmd := exec.Command("npm", "install", depName, "-g", "--registry", url) + outputBytes, err := cmd.Output() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return fmt.Sprintf("error: %s", err.Error()), err + } + return string(outputBytes), nil +} + +func InstallDepRemote(nodeId string, lang string, depName string) (err error) { + params := make(map[string]string) + params["lang"] = lang + params["dep_name"] = depName + s := GetService(entity.RpcMessage{ + NodeId: nodeId, + Method: constants.RpcInstallDep, + Params: params, + Timeout: 300, + }) + _, err = s.ClientHandle() + if err != nil { + return + } + return +} diff --git a/backend/services/rpc/install_lang.go b/backend/services/rpc/install_lang.go new file mode 100644 index 00000000..39f5a6d8 --- /dev/null +++ b/backend/services/rpc/install_lang.go @@ -0,0 +1,73 @@ +package rpc + +import ( + "crawlab/constants" + "crawlab/entity" + "crawlab/utils" + "errors" + "fmt" + "github.com/apex/log" + "os/exec" + "path" + "runtime/debug" +) + +type InstallLangService struct { + msg entity.RpcMessage +} + +func (s *InstallLangService) ServerHandle() (entity.RpcMessage, error) { + lang := utils.GetRpcParam("lang", s.msg.Params) + output, err := InstallLangLocal(lang) + s.msg.Result = output + if err != nil { + s.msg.Error = err.Error() + return s.msg, err + } + return s.msg, nil +} + +func (s *InstallLangService) ClientHandle() (o interface{}, err error) { + // 发起 RPC 请求，获取服务端数据 + go func() { + _, err := ClientFunc(s.msg)() + if err != nil { + return + } + }() + + return +} + +// 本地安装语言 +func InstallLangLocal(lang string) (o string, err error) { + l := utils.GetLangFromLangNamePlain(lang) + if l.Name == "" || l.InstallScript == "" { + return "", errors.New(fmt.Sprintf("%s is not implemented", lang)) + } + cmd := exec.Command("/bin/sh", path.Join("scripts", l.InstallScript)) + output, err := cmd.Output() + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return string(output), err + } + return +} + +// 远端安装语言 +func InstallLangRemote(nodeId string, lang string) (o string, err error) { + params := make(map[string]string) + params["lang"] = lang + s := GetService(entity.RpcMessage{ + NodeId: nodeId, + Method: constants.RpcInstallLang, + Params: params, + Timeout: 60, + }) + _, err = s.ClientHandle() + if err != nil { + return + } + return +} diff --git a/backend/services/rpc/uninstall_dep.go b/backend/services/rpc/uninstall_dep.go new file mode 100644 index 00000000..1b8b8ecb --- /dev/null +++ b/backend/services/rpc/uninstall_dep.go @@ -0,0 +1,96 @@ +package rpc + +import ( + "crawlab/constants" + "crawlab/entity" + "crawlab/utils" + "errors" + "fmt" + "github.com/apex/log" + "os/exec" + "runtime/debug" +) + +type UninstallDepService struct { + msg entity.RpcMessage +} + +func (s *UninstallDepService) ServerHandle() (entity.RpcMessage, error) { + lang := utils.GetRpcParam("lang", s.msg.Params) + depName := utils.GetRpcParam("dep_name", s.msg.Params) + if err := UninstallDepLocal(lang, depName); err != nil { + s.msg.Error = err.Error() + return s.msg, err + } + s.msg.Result = "success" + return s.msg, nil +} + +func (s *UninstallDepService) ClientHandle() (o interface{}, err error) { + // 发起 RPC 请求，获取服务端数据 + _, err = ClientFunc(s.msg)() + if err != nil { + return + } + + return +} + +func UninstallDepLocal(lang string, depName string) error { + if lang == constants.Python { + output, err := UninstallPythonDepLocal(depName) + if err != nil { + log.Debugf(output) + return err + } + } else if lang == constants.Nodejs { + output, err := UninstallNodejsDepLocal(depName) + if err != nil { + log.Debugf(output) + return err + } + } else { + return errors.New(fmt.Sprintf("%s is not implemented", lang)) + } + return nil +} + +func UninstallPythonDepLocal(depName string) (string, error) { + cmd := exec.Command("pip", "uninstall", "-y", depName) + outputBytes, err := cmd.Output() + if err != nil { + log.Errorf(string(outputBytes)) + log.Errorf(err.Error()) + debug.PrintStack() + return fmt.Sprintf("error: %s", err.Error()), err + } + return string(outputBytes), nil +} + +func UninstallNodejsDepLocal(depName string) (string, error) { + cmd := exec.Command("npm", "uninstall", depName, "-g") + outputBytes, err := cmd.Output() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return fmt.Sprintf("error: %s", err.Error()), err + } + return string(outputBytes), nil +} + +func UninstallDepRemote(nodeId string, lang string, depName string) (err error) { + params := make(map[string]string) + params["lang"] = lang + params["dep_name"] = depName + s := GetService(entity.RpcMessage{ + NodeId: nodeId, + Method: constants.RpcUninstallDep, + Params: params, + Timeout: 300, + }) + _, err = s.ClientHandle() + if err != nil { + return + } + return +} diff --git a/backend/services/spider.go b/backend/services/spider.go index 392b0488..77bb19ab 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -412,6 +412,111 @@ func CopySpider(spider model.Spider, newName string) error { return nil } +func InitDemoSpiders () { + // 添加Demo爬虫 + templateSpidersDir := "./template/spiders" + for _, info := range utils.ListDir(templateSpidersDir) { + if !info.IsDir() { + continue + } + spiderName := info.Name() + + // 如果爬虫在数据库中不存在，则添加 + spider := model.GetSpiderByName(spiderName) + if spider.Name != "" { + // 存在同名爬虫，跳过 + continue + } + + // 拷贝爬虫 + templateSpiderPath := path.Join(templateSpidersDir, spiderName) + spiderPath := path.Join(viper.GetString("spider.path"), spiderName) + if utils.Exists(spiderPath) { + utils.RemoveFiles(spiderPath) + } + if err := utils.CopyDir(templateSpiderPath, spiderPath); err != nil { + log.Errorf("copy error: " + err.Error()) + debug.PrintStack() + continue + } + + // 构造配置数据 + configData := entity.ConfigSpiderData{} + + // 读取YAML文件 + yamlFile, err := ioutil.ReadFile(path.Join(spiderPath, "Spiderfile")) + if err != nil { + log.Errorf("read yaml error: " + err.Error()) + //debug.PrintStack() + continue + } + + // 反序列化 + if err := yaml.Unmarshal(yamlFile, &configData); err != nil { + log.Errorf("unmarshal error: " + err.Error()) + debug.PrintStack() + continue + } + + if configData.Type == constants.Customized { + // 添加该爬虫到数据库 + spider = model.Spider{ + Id: bson.NewObjectId(), + Name: spiderName, + DisplayName: configData.DisplayName, + Type: constants.Customized, + Col: configData.Col, + Src: spiderPath, + Remark: configData.Remark, + ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), + FileId: bson.ObjectIdHex(constants.ObjectIdNull), + Cmd: configData.Cmd, + } + if err := spider.Add(); err != nil { + log.Errorf("add spider error: " + err.Error()) + debug.PrintStack() + continue + } + + // 上传爬虫到GridFS + if err := UploadSpiderToGridFsFromMaster(spider); err != nil { + log.Errorf("upload spider error: " + err.Error()) + debug.PrintStack() + continue + } + } else if configData.Type == constants.Configurable || configData.Type == "config" { + // 添加该爬虫到数据库 + spider = model.Spider{ + Id: bson.NewObjectId(), + Name: configData.Name, + DisplayName: configData.DisplayName, + Type: constants.Configurable, + Col: configData.Col, + Src: spiderPath, + Remark: configData.Remark, + ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), + FileId: bson.ObjectIdHex(constants.ObjectIdNull), + Config: configData, + } + if err := spider.Add(); err != nil { + log.Errorf("add spider error: " + err.Error()) + debug.PrintStack() + continue + } + + // 根据序列化后的数据处理爬虫文件 + if err := ProcessSpiderFilesFromConfigData(spider, configData); err != nil { + log.Errorf("add spider error: " + err.Error()) + debug.PrintStack() + continue + } + } + } + + // 发布所有爬虫 + PublishAllSpiders() +} + // 启动爬虫服务 func InitSpiderService() error { // 构造定时任务执行器 @@ -423,110 +528,12 @@ func InitSpiderService() error { // 启动定时任务 cPub.Start() + if model.IsMaster() && viper.GetString("setting.demoSpiders") == "Y" { + // 初始化Demo爬虫 + InitDemoSpiders() + } + if model.IsMaster() { - // 添加Demo爬虫 - templateSpidersDir := "./template/spiders" - for _, info := range utils.ListDir(templateSpidersDir) { - if !info.IsDir() { - continue - } - spiderName := info.Name() - - // 如果爬虫在数据库中不存在，则添加 - spider := model.GetSpiderByName(spiderName) - if spider.Name != "" { - // 存在同名爬虫，跳过 - continue - } - - // 拷贝爬虫 - templateSpiderPath := path.Join(templateSpidersDir, spiderName) - spiderPath := path.Join(viper.GetString("spider.path"), spiderName) - if utils.Exists(spiderPath) { - utils.RemoveFiles(spiderPath) - } - if err := utils.CopyDir(templateSpiderPath, spiderPath); err != nil { - log.Errorf("copy error: " + err.Error()) - debug.PrintStack() - continue - } - - // 构造配置数据 - configData := entity.ConfigSpiderData{} - - // 读取YAML文件 - yamlFile, err := ioutil.ReadFile(path.Join(spiderPath, "Spiderfile")) - if err != nil { - log.Errorf("read yaml error: " + err.Error()) - //debug.PrintStack() - continue - } - - // 反序列化 - if err := yaml.Unmarshal(yamlFile, &configData); err != nil { - log.Errorf("unmarshal error: " + err.Error()) - debug.PrintStack() - continue - } - - if configData.Type == constants.Customized { - // 添加该爬虫到数据库 - spider = model.Spider{ - Id: bson.NewObjectId(), - Name: spiderName, - DisplayName: configData.DisplayName, - Type: constants.Customized, - Col: configData.Col, - Src: spiderPath, - Remark: configData.Remark, - ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), - FileId: bson.ObjectIdHex(constants.ObjectIdNull), - Cmd: configData.Cmd, - } - if err := spider.Add(); err != nil { - log.Errorf("add spider error: " + err.Error()) - debug.PrintStack() - continue - } - - // 上传爬虫到GridFS - if err := UploadSpiderToGridFsFromMaster(spider); err != nil { - log.Errorf("upload spider error: " + err.Error()) - debug.PrintStack() - continue - } - } else if configData.Type == constants.Configurable || configData.Type == "config" { - // 添加该爬虫到数据库 - spider = model.Spider{ - Id: bson.NewObjectId(), - Name: configData.Name, - DisplayName: configData.DisplayName, - Type: constants.Configurable, - Col: configData.Col, - Src: spiderPath, - Remark: configData.Remark, - ProjectId: bson.ObjectIdHex(constants.ObjectIdNull), - FileId: bson.ObjectIdHex(constants.ObjectIdNull), - Config: configData, - } - if err := spider.Add(); err != nil { - log.Errorf("add spider error: " + err.Error()) - debug.PrintStack() - continue - } - - // 根据序列化后的数据处理爬虫文件 - if err := ProcessSpiderFilesFromConfigData(spider, configData); err != nil { - log.Errorf("add spider error: " + err.Error()) - debug.PrintStack() - continue - } - } - } - - // 发布所有爬虫 - PublishAllSpiders() - // 构造 Git 定时任务 GitCron = &GitCronScheduler{ cron: cron.New(cron.WithSeconds()), diff --git a/backend/services/system.go b/backend/services/system.go index 285cad5a..fd9bb83d 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -6,6 +6,7 @@ import ( "crawlab/entity" "crawlab/lib/cron" "crawlab/model" + "crawlab/services/rpc" "crawlab/utils" "encoding/json" "errors" @@ -13,7 +14,6 @@ import ( "github.com/apex/log" "github.com/imroc/req" "os/exec" - "path" "regexp" "runtime/debug" "sort" @@ -64,42 +64,25 @@ func GetSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) { // 获取语言列表 func GetLangList(nodeId string) []entity.Lang { - list := []entity.Lang{ - {Name: "Python", ExecutableName: "python", ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, DepExecutablePath: "/usr/local/bin/pip"}, - {Name: "Node.js", ExecutableName: "node", ExecutablePaths: []string{"/usr/bin/node", "/usr/local/bin/node"}, DepExecutablePath: "/usr/local/bin/npm"}, - //{Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}}, - } + list := utils.GetLangList() for i, lang := range list { - list[i].Installed = IsInstalledLang(nodeId, lang) + status, _ := GetLangInstallStatus(nodeId, lang) + list[i].InstallStatus = status } return list } -// 根据语言名获取语言实例 -func GetLangFromLangName(nodeId string, name string) entity.Lang { - langList := GetLangList(nodeId) - for _, lang := range langList { - if lang.ExecutableName == name { - return lang +func GetLangInstallStatus(nodeId string, lang entity.Lang) (string, error) { + if IsMasterNode(nodeId) { + lang := rpc.GetLangLocal(lang) + return lang.InstallStatus, nil + } else { + lang, err := rpc.GetLangRemote(nodeId, lang) + if err != nil { + return "", err } + return lang.InstallStatus, nil } - return entity.Lang{} -} - -// 是否已安装该依赖 -func IsInstalledLang(nodeId string, lang entity.Lang) bool { - sysInfo, err := GetSystemInfo(nodeId) - if err != nil { - return false - } - for _, exec := range sysInfo.Executables { - for _, path := range lang.ExecutablePaths { - if exec.Path == path { - return true - } - } - } - return false } // 是否已安装该依赖 @@ -112,6 +95,8 @@ func IsInstalledDep(installedDepList []entity.Dependency, dep entity.Dependency) return false } +// ========Python======== + // 初始化函数 func InitDepsFetcher() error { c := cron.New(cron.WithSeconds()) @@ -126,10 +111,6 @@ func InitDepsFetcher() error { return nil } -// ========= -// Python -// ========= - type PythonDepJsonData struct { Info PythonDepJsonDataInfo `json:"info"` } @@ -183,12 +164,12 @@ func GetPythonDepList(nodeId string, searchDepName string) ([]entity.Dependency, // 获取已安装依赖列表 var installedDepList []entity.Dependency if IsMasterNode(nodeId) { - installedDepList, err = GetPythonLocalInstalledDepList(nodeId) + installedDepList, err = rpc.GetInstalledDepsLocal(constants.Python) if err != nil { return list, err } } else { - installedDepList, err = GetPythonRemoteInstalledDepList(nodeId) + installedDepList, err = rpc.GetInstalledDepsRemote(nodeId, constants.Python) if err != nil { return list, err } @@ -351,205 +332,9 @@ func UpdatePythonDepList() { } } -// 获取Python本地已安装的依赖列表 -func GetPythonLocalInstalledDepList(nodeId string) ([]entity.Dependency, error) { - var list []entity.Dependency +// ========./Python======== - lang := GetLangFromLangName(nodeId, constants.Python) - if !IsInstalledLang(nodeId, lang) { - return list, errors.New("python is not installed") - } - cmd := exec.Command("pip", "freeze") - outputBytes, err := cmd.Output() - if err != nil { - debug.PrintStack() - return list, err - } - - for _, line := range strings.Split(string(outputBytes), "\n") { - arr := strings.Split(line, "==") - if len(arr) < 2 { - continue - } - dep := entity.Dependency{ - Name: strings.ToLower(arr[0]), - Version: arr[1], - Installed: true, - } - list = append(list, dep) - } - - return list, nil -} - -// 获取Python远端依赖列表 -func GetPythonRemoteInstalledDepList(nodeId string) ([]entity.Dependency, error) { - depList, err := RpcClientGetInstalledDepList(nodeId, constants.Python) - if err != nil { - return depList, err - } - return depList, nil -} - -// 安装Python本地依赖 -func InstallPythonLocalDep(depName string) (string, error) { - // 依赖镜像URL - url := "https://pypi.tuna.tsinghua.edu.cn/simple" - - cmd := exec.Command("pip", "install", depName, "-i", url) - outputBytes, err := cmd.Output() - if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return fmt.Sprintf("error: %s", err.Error()), err - } - return string(outputBytes), nil -} - -// 获取Python远端依赖列表 -func InstallPythonRemoteDep(nodeId string, depName string) (string, error) { - output, err := RpcClientInstallDep(nodeId, constants.Python, depName) - if err != nil { - return output, err - } - return output, nil -} - -// 安装Python本地依赖 -func UninstallPythonLocalDep(depName string) (string, error) { - cmd := exec.Command("pip", "uninstall", "-y", depName) - outputBytes, err := cmd.Output() - if err != nil { - log.Errorf(string(outputBytes)) - log.Errorf(err.Error()) - debug.PrintStack() - return fmt.Sprintf("error: %s", err.Error()), err - } - return string(outputBytes), nil -} - -// 获取Python远端依赖列表 -func UninstallPythonRemoteDep(nodeId string, depName string) (string, error) { - output, err := RpcClientUninstallDep(nodeId, constants.Python, depName) - if err != nil { - return output, err - } - return output, nil -} - -// ============== -// Node.js -// ============== - -func InstallNodejsLocalLang() (string, error) { - cmd := exec.Command("/bin/sh", path.Join("scripts", "install-nodejs.sh")) - output, err := cmd.Output() - if err != nil { - log.Error(err.Error()) - debug.PrintStack() - return string(output), err - } - - // TODO: check if Node.js is installed successfully - - return string(output), nil -} - -// 获取Node.js远端依赖列表 -func InstallNodejsRemoteLang(nodeId string) (string, error) { - output, err := RpcClientInstallLang(nodeId, constants.Nodejs) - if err != nil { - return output, err - } - return output, nil -} - -// 获取Nodejs本地已安装的依赖列表 -func GetNodejsLocalInstalledDepList(nodeId string) ([]entity.Dependency, error) { - var list []entity.Dependency - - lang := GetLangFromLangName(nodeId, constants.Nodejs) - if !IsInstalledLang(nodeId, lang) { - return list, errors.New("nodejs is not installed") - } - cmd := exec.Command("npm", "ls", "-g", "--depth", "0") - outputBytes, _ := cmd.Output() - //if err != nil { - // log.Error("error: " + string(outputBytes)) - // debug.PrintStack() - // return list, err - //} - - regex := regexp.MustCompile("\\s(.*)@(.*)") - for _, line := range strings.Split(string(outputBytes), "\n") { - arr := regex.FindStringSubmatch(line) - if len(arr) < 3 { - continue - } - dep := entity.Dependency{ - Name: strings.ToLower(arr[1]), - Version: arr[2], - Installed: true, - } - list = append(list, dep) - } - - return list, nil -} - -// 获取Nodejs远端依赖列表 -func GetNodejsRemoteInstalledDepList(nodeId string) ([]entity.Dependency, error) { - depList, err := RpcClientGetInstalledDepList(nodeId, constants.Nodejs) - if err != nil { - return depList, err - } - return depList, nil -} - -// 安装Nodejs本地依赖 -func InstallNodejsLocalDep(depName string) (string, error) { - // 依赖镜像URL - url := "https://registry.npm.taobao.org" - - cmd := exec.Command("npm", "install", depName, "-g", "--registry", url) - outputBytes, err := cmd.Output() - if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return fmt.Sprintf("error: %s", err.Error()), err - } - return string(outputBytes), nil -} - -// 获取Nodejs远端依赖列表 -func InstallNodejsRemoteDep(nodeId string, depName string) (string, error) { - output, err := RpcClientInstallDep(nodeId, constants.Nodejs, depName) - if err != nil { - return output, err - } - return output, nil -} - -// 安装Nodejs本地依赖 -func UninstallNodejsLocalDep(depName string) (string, error) { - cmd := exec.Command("npm", "uninstall", depName, "-g") - outputBytes, err := cmd.Output() - if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return fmt.Sprintf("error: %s", err.Error()), err - } - return string(outputBytes), nil -} - -// 获取Nodejs远端依赖列表 -func UninstallNodejsRemoteDep(nodeId string, depName string) (string, error) { - output, err := RpcClientUninstallDep(nodeId, constants.Nodejs, depName) - if err != nil { - return output, err - } - return output, nil -} +// ========Node.js======== // 获取Nodejs本地依赖列表 func GetNodejsDepList(nodeId string, searchDepName string) (depList []entity.Dependency, err error) { @@ -560,12 +345,12 @@ func GetNodejsDepList(nodeId string, searchDepName string) (depList []entity.Dep // 获取已安装依赖列表 var installedDepList []entity.Dependency if IsMasterNode(nodeId) { - installedDepList, err = GetNodejsLocalInstalledDepList(nodeId) + installedDepList, err = rpc.GetInstalledDepsLocal(constants.Nodejs) if err != nil { return depList, err } } else { - installedDepList, err = GetNodejsRemoteInstalledDepList(nodeId) + installedDepList, err = rpc.GetInstalledDepsRemote(nodeId, constants.Nodejs) if err != nil { return depList, err } @@ -585,3 +370,5 @@ func GetNodejsDepList(nodeId string, searchDepName string) (depList []entity.Dep return depList, nil } + +// ========./Node.js======== diff --git a/backend/services/task.go b/backend/services/task.go index f9bab5ce..469fa8da 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -852,19 +852,19 @@ func SendNotifications(u model.User, t model.Task, s model.Spider) { } } -func UnlockLongTask(s model.Spider, n model.Node) { - if s.IsLongTask { - colName := "long-tasks" - key := fmt.Sprintf("%s:%s", s.Id.Hex(), n.Id.Hex()) - _ = database.RedisClient.HDel(colName, key) - } -} - func InitTaskExecutor() error { + // 构造任务执行器 c := cron.New(cron.WithSeconds()) Exec = &Executor{ Cron: c, } + + // 如果不允许主节点运行任务，则跳过 + if model.IsMaster() && viper.GetString("setting.runOnMaster") == "N" { + return nil + } + + // 运行定时任务 if err := Exec.Start(); err != nil { return err } diff --git a/backend/utils/rpc.go b/backend/utils/rpc.go new file mode 100644 index 00000000..03414199 --- /dev/null +++ b/backend/utils/rpc.go @@ -0,0 +1,14 @@ +package utils + +import "encoding/json" + +// Object 转化为 String +func ObjectToString(params interface{}) string { + bytes, _ := json.Marshal(params) + return BytesToString(bytes) +} + +// 获取 RPC 参数 +func GetRpcParam(key string, params map[string]string) string { + return params[key] +} diff --git a/backend/utils/system.go b/backend/utils/system.go new file mode 100644 index 00000000..f8f917be --- /dev/null +++ b/backend/utils/system.go @@ -0,0 +1,62 @@ +package utils + +import "crawlab/entity" + +func GetLangList() []entity.Lang { + list := []entity.Lang{ + { + Name: "Python", + ExecutableName: "python", + ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, + DepExecutablePath: "/usr/local/bin/pip", + LockPath: "/tmp/install-python.lock", + }, + { + Name: "Node.js", + ExecutableName: "node", + ExecutablePaths: []string{"/usr/bin/node", "/usr/local/bin/node"}, + DepExecutablePath: "/usr/local/bin/npm", + LockPath: "/tmp/install-nodejs.lock", + InstallScript: "install-nodejs.sh", + }, + { + Name: "Java", + ExecutableName: "java", + ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}, + LockPath: "/tmp/install-java.lock", + InstallScript: "install-java.sh", + }, + { + Name: ".Net Core", + ExecutableName: "dotnet", + ExecutablePaths: []string{"/usr/bin/dotnet", "/usr/local/bin/dotnet"}, + LockPath: "/tmp/install-dotnet.lock", + InstallScript: "install-dotnet.sh", + }, + { + Name: "PHP", + ExecutableName: "php", + ExecutablePaths: []string{"/usr/bin/php", "/usr/local/bin/php"}, + LockPath: "/tmp/install-php.lock", + InstallScript: "install-php.sh", + }, + } + return list +} + +// 获取语言列表 +func GetLangListPlain() []entity.Lang { + list := GetLangList() + return list +} + +// 根据语言名获取语言实例，不包含状态 +func GetLangFromLangNamePlain(name string) entity.Lang { + langList := GetLangListPlain() + for _, lang := range langList { + if lang.ExecutableName == name { + return lang + } + } + return entity.Lang{} +} diff --git a/devops/develop/crawlab-master.yaml b/devops/develop/crawlab-master.yaml index 1168c404..b2ea07b2 100644 --- a/devops/develop/crawlab-master.yaml +++ b/devops/develop/crawlab-master.yaml @@ -13,13 +13,12 @@ spec: type: NodePort --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: crawlab-master namespace: crawlab-develop spec: - strategy: - type: Recreate + serviceName: crawlab-master selector: matchLabels: app: crawlab-master @@ -42,9 +41,13 @@ spec: - name: CRAWLAB_SETTING_ALLOWREGISTER value: "Y" - name: CRAWLAB_SERVER_LANG_NODE - value: "Y" + value: "N" - name: CRAWLAB_SERVER_LANG_JAVA - value: "Y" + value: "N" + - name: CRAWLAB_SERVER_LANG_DOTNET + value: "N" + - name: CRAWLAB_SERVER_REGISTER_TYPE + value: "hostname" ports: - containerPort: 8080 name: crawlab \ No newline at end of file diff --git a/devops/develop/crawlab-worker.yaml b/devops/develop/crawlab-worker.yaml index 096f0ccc..037aa8fc 100644 --- a/devops/develop/crawlab-worker.yaml +++ b/devops/develop/crawlab-worker.yaml @@ -1,12 +1,11 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: crawlab-worker namespace: crawlab-develop spec: + serviceName: crawlab-worker replicas: 2 - strategy: - type: Recreate selector: matchLabels: app: crawlab-worker @@ -27,7 +26,10 @@ spec: - name: CRAWLAB_REDIS_ADDRESS value: "redis" - name: CRAWLAB_SERVER_LANG_NODE - value: "Y" + value: "N" - name: CRAWLAB_SERVER_LANG_JAVA - value: "Y" - \ No newline at end of file + value: "N" + - name: CRAWLAB_SERVER_LANG_DOTNET + value: "N" + - name: CRAWLAB_SERVER_REGISTER_TYPE + value: "hostname" diff --git a/devops/release/crawlab-master.yaml b/devops/release/crawlab-master.yaml index 342d1300..229a7cdd 100644 --- a/devops/release/crawlab-master.yaml +++ b/devops/release/crawlab-master.yaml @@ -13,13 +13,12 @@ spec: type: NodePort --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: crawlab-master namespace: crawlab-release spec: - strategy: - type: Recreate + serviceName: crawlab-master selector: matchLabels: app: crawlab-master @@ -42,9 +41,13 @@ spec: - name: CRAWLAB_SETTING_ALLOWREGISTER value: "Y" - name: CRAWLAB_SERVER_LANG_NODE - value: "Y" + value: "N" - name: CRAWLAB_SERVER_LANG_JAVA - value: "Y" + value: "N" + - name: CRAWLAB_SERVER_LANG_DOTNET + value: "N" + - name: CRAWLAB_SERVER_REGISTER_TYPE + value: "hostname" ports: - containerPort: 8080 name: crawlab \ No newline at end of file diff --git a/devops/release/crawlab-worker.yaml b/devops/release/crawlab-worker.yaml index bf96607e..fd6772af 100644 --- a/devops/release/crawlab-worker.yaml +++ b/devops/release/crawlab-worker.yaml @@ -1,12 +1,11 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: crawlab-worker namespace: crawlab-release spec: + serviceName: crawlab-worker replicas: 2 - strategy: - type: Recreate selector: matchLabels: app: crawlab-worker @@ -27,6 +26,10 @@ spec: - name: CRAWLAB_REDIS_ADDRESS value: "redis" - name: CRAWLAB_SERVER_LANG_NODE - value: "Y" + value: "N" - name: CRAWLAB_SERVER_LANG_JAVA - value: "Y" + value: "N" + - name: CRAWLAB_SERVER_LANG_DOTNET + value: "N" + - name: CRAWLAB_SERVER_REGISTER_TYPE + value: "hostname" diff --git a/docker-compose.yml b/docker-compose.yml index 26cbb989..e9cb08cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,11 +21,15 @@ services: # CRAWLAB_LOG_DELETEFREQUENCY: "@hourly" # frequency of deleting log files 删除日志文件的频率. 默认为每小时 # CRAWLAB_SERVER_REGISTER_TYPE: "mac" # node register type 节点注册方式. 默认为 mac 地址，也可设置为 ip（防止 mac 地址冲突） # CRAWLAB_SERVER_REGISTER_IP: "127.0.0.1" # node register ip 节点注册IP. 节点唯一识别号，只有当 CRAWLAB_SERVER_REGISTER_TYPE 为 "ip" 时才生效 - # CRAWLAB_TASK_WORKERS: 4 # number of task executors 任务执行器个数（并行执行任务数） + # CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数（并行执行任务数） # CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境 # CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境 + # CRAWLAB_SERVER_LANG_DOTNET: "Y" # whether to pre-install .Net core 预安装 .Net Core 语言环境 + # CRAWLAB_SERVER_LANG_PHP: "Y" # whether to pre-install PHP 预安装 PHP 语言环境 # CRAWLAB_SETTING_ALLOWREGISTER: "N" # whether to allow user registration 是否允许用户注册 # CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程 + # CRAWLAB_SETTING_RUNONMASTER: "N" # whether to run on master node 是否在主节点上运行任务 + # CRAWLAB_SETTING_DEMOSPIDERS: "Y" # whether to init demo spiders 是否使用Demo爬虫 # CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址 # CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口 # CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱 diff --git a/docker_init.sh b/docker_init.sh index 19cf8471..dce1137a 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -22,18 +22,12 @@ fi # start nginx service nginx start -# install languages: Node.js -if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; +# install languages +if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ]; then - echo "installing node.js" - /bin/sh /app/backend/scripts/install-nodejs.sh >> /var/log/install-nodejs.sh.log 2>&1 & -fi - -# install languages: Java -if [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ]; -then - echo "installing java" - /bin/sh /app/backend/scripts/install-java.sh >> /var/log/install-java.sh.log 2>&1 & + echo "installing languages" + echo "you can view log at /var/log/install.sh.log" + /bin/sh /app/backend/scripts/install.sh >> /var/log/install.sh.log 2>&1 & fi # generate ssh diff --git a/frontend/package.json b/frontend/package.json index 61eba954..3bddca53 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "crawlab", - "version": "0.4.7", + "version": "0.4.8", "private": true, "scripts": { "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development", diff --git a/frontend/public/font-awesome.min.css b/frontend/public/font-awesome.min.css new file mode 100644 index 00000000..540440ce --- /dev/null +++ b/frontend/public/font-awesome.min.css @@ -0,0 +1,4 @@ +/*! + * Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome + * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) + */@font-face{font-family:'FontAwesome';src:url('../fonts/fontawesome-webfont.eot?v=4.7.0');src:url('../fonts/fontawesome-webfont.eot?#iefix&v=4.7.0') format('embedded-opentype'),url('../fonts/fontawesome-webfont.woff2?v=4.7.0') format('woff2'),url('../fonts/fontawesome-webfont.woff?v=4.7.0') format('woff'),url('../fonts/fontawesome-webfont.ttf?v=4.7.0') format('truetype'),url('../fonts/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular') format('svg');font-weight:normal;font-style:normal}.fa{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left{margin-right:.3em}.fa.fa-pull-right{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scale(-1, 1);-ms-transform:scale(-1, 1);transform:scale(-1, 1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scale(1, -1);-ms-transform:scale(1, -1);transform:scale(1, -1)}:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-flip-horizontal,:root .fa-flip-vertical{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-remove:before,.fa-close:before,.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-gear:before,.fa-cog:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-rotate-right:before,.fa-repeat:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-photo:before,.fa-image:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before,.fa-bar-chart:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-gears:before,.fa-cogs:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook-f:before,.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-feed:before,.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-save:before,.fa-floppy-o:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-navicon:before,.fa-reorder:before,.fa-bars:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-unsorted:before,.fa-sort:before{content:"\f0dc"}.fa-sort-down:before,.fa-sort-desc:before{content:"\f0dd"}.fa-sort-up:before,.fa-sort-asc:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-legal:before,.fa-gavel:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-flash:before,.fa-bolt:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-paste:before,.fa-clipboard:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-unlink:before,.fa-chain-broken:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"}.fa-euro:before,.fa-eur:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-rupee:before,.fa-inr:before{content:"\f156"}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"}.fa-won:before,.fa-krw:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before,.fa-gratipay:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-turkish-lira:before,.fa-try:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-institution:before,.fa-bank:before,.fa-university:before{content:"\f19c"}.fa-mortar-board:before,.fa-graduation-cap:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper-pp:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-photo-o:before,.fa-file-picture-o:before,.fa-file-image-o:before{content:"\f1c5"}.fa-file-zip-o:before,.fa-file-archive-o:before{content:"\f1c6"}.fa-file-sound-o:before,.fa-file-audio-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-saver:before,.fa-support:before,.fa-life-ring:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-resistance:before,.fa-rebel:before{content:"\f1d0"}.fa-ge:before,.fa-empire:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-y-combinator-square:before,.fa-yc-square:before,.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-send:before,.fa-paper-plane:before{content:"\f1d8"}.fa-send-o:before,.fa-paper-plane-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.fa-soccer-ball-o:before,.fa-futbol-o:before{content:"\f1e3"}.fa-tty:before{content:"\f1e4"}.fa-binoculars:before{content:"\f1e5"}.fa-plug:before{content:"\f1e6"}.fa-slideshare:before{content:"\f1e7"}.fa-twitch:before{content:"\f1e8"}.fa-yelp:before{content:"\f1e9"}.fa-newspaper-o:before{content:"\f1ea"}.fa-wifi:before{content:"\f1eb"}.fa-calculator:before{content:"\f1ec"}.fa-paypal:before{content:"\f1ed"}.fa-google-wallet:before{content:"\f1ee"}.fa-cc-visa:before{content:"\f1f0"}.fa-cc-mastercard:before{content:"\f1f1"}.fa-cc-discover:before{content:"\f1f2"}.fa-cc-amex:before{content:"\f1f3"}.fa-cc-paypal:before{content:"\f1f4"}.fa-cc-stripe:before{content:"\f1f5"}.fa-bell-slash:before{content:"\f1f6"}.fa-bell-slash-o:before{content:"\f1f7"}.fa-trash:before{content:"\f1f8"}.fa-copyright:before{content:"\f1f9"}.fa-at:before{content:"\f1fa"}.fa-eyedropper:before{content:"\f1fb"}.fa-paint-brush:before{content:"\f1fc"}.fa-birthday-cake:before{content:"\f1fd"}.fa-area-chart:before{content:"\f1fe"}.fa-pie-chart:before{content:"\f200"}.fa-line-chart:before{content:"\f201"}.fa-lastfm:before{content:"\f202"}.fa-lastfm-square:before{content:"\f203"}.fa-toggle-off:before{content:"\f204"}.fa-toggle-on:before{content:"\f205"}.fa-bicycle:before{content:"\f206"}.fa-bus:before{content:"\f207"}.fa-ioxhost:before{content:"\f208"}.fa-angellist:before{content:"\f209"}.fa-cc:before{content:"\f20a"}.fa-shekel:before,.fa-sheqel:before,.fa-ils:before{content:"\f20b"}.fa-meanpath:before{content:"\f20c"}.fa-buysellads:before{content:"\f20d"}.fa-connectdevelop:before{content:"\f20e"}.fa-dashcube:before{content:"\f210"}.fa-forumbee:before{content:"\f211"}.fa-leanpub:before{content:"\f212"}.fa-sellsy:before{content:"\f213"}.fa-shirtsinbulk:before{content:"\f214"}.fa-simplybuilt:before{content:"\f215"}.fa-skyatlas:before{content:"\f216"}.fa-cart-plus:before{content:"\f217"}.fa-cart-arrow-down:before{content:"\f218"}.fa-diamond:before{content:"\f219"}.fa-ship:before{content:"\f21a"}.fa-user-secret:before{content:"\f21b"}.fa-motorcycle:before{content:"\f21c"}.fa-street-view:before{content:"\f21d"}.fa-heartbeat:before{content:"\f21e"}.fa-venus:before{content:"\f221"}.fa-mars:before{content:"\f222"}.fa-mercury:before{content:"\f223"}.fa-intersex:before,.fa-transgender:before{content:"\f224"}.fa-transgender-alt:before{content:"\f225"}.fa-venus-double:before{content:"\f226"}.fa-mars-double:before{content:"\f227"}.fa-venus-mars:before{content:"\f228"}.fa-mars-stroke:before{content:"\f229"}.fa-mars-stroke-v:before{content:"\f22a"}.fa-mars-stroke-h:before{content:"\f22b"}.fa-neuter:before{content:"\f22c"}.fa-genderless:before{content:"\f22d"}.fa-facebook-official:before{content:"\f230"}.fa-pinterest-p:before{content:"\f231"}.fa-whatsapp:before{content:"\f232"}.fa-server:before{content:"\f233"}.fa-user-plus:before{content:"\f234"}.fa-user-times:before{content:"\f235"}.fa-hotel:before,.fa-bed:before{content:"\f236"}.fa-viacoin:before{content:"\f237"}.fa-train:before{content:"\f238"}.fa-subway:before{content:"\f239"}.fa-medium:before{content:"\f23a"}.fa-yc:before,.fa-y-combinator:before{content:"\f23b"}.fa-optin-monster:before{content:"\f23c"}.fa-opencart:before{content:"\f23d"}.fa-expeditedssl:before{content:"\f23e"}.fa-battery-4:before,.fa-battery:before,.fa-battery-full:before{content:"\f240"}.fa-battery-3:before,.fa-battery-three-quarters:before{content:"\f241"}.fa-battery-2:before,.fa-battery-half:before{content:"\f242"}.fa-battery-1:before,.fa-battery-quarter:before{content:"\f243"}.fa-battery-0:before,.fa-battery-empty:before{content:"\f244"}.fa-mouse-pointer:before{content:"\f245"}.fa-i-cursor:before{content:"\f246"}.fa-object-group:before{content:"\f247"}.fa-object-ungroup:before{content:"\f248"}.fa-sticky-note:before{content:"\f249"}.fa-sticky-note-o:before{content:"\f24a"}.fa-cc-jcb:before{content:"\f24b"}.fa-cc-diners-club:before{content:"\f24c"}.fa-clone:before{content:"\f24d"}.fa-balance-scale:before{content:"\f24e"}.fa-hourglass-o:before{content:"\f250"}.fa-hourglass-1:before,.fa-hourglass-start:before{content:"\f251"}.fa-hourglass-2:before,.fa-hourglass-half:before{content:"\f252"}.fa-hourglass-3:before,.fa-hourglass-end:before{content:"\f253"}.fa-hourglass:before{content:"\f254"}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:"\f255"}.fa-hand-stop-o:before,.fa-hand-paper-o:before{content:"\f256"}.fa-hand-scissors-o:before{content:"\f257"}.fa-hand-lizard-o:before{content:"\f258"}.fa-hand-spock-o:before{content:"\f259"}.fa-hand-pointer-o:before{content:"\f25a"}.fa-hand-peace-o:before{content:"\f25b"}.fa-trademark:before{content:"\f25c"}.fa-registered:before{content:"\f25d"}.fa-creative-commons:before{content:"\f25e"}.fa-gg:before{content:"\f260"}.fa-gg-circle:before{content:"\f261"}.fa-tripadvisor:before{content:"\f262"}.fa-odnoklassniki:before{content:"\f263"}.fa-odnoklassniki-square:before{content:"\f264"}.fa-get-pocket:before{content:"\f265"}.fa-wikipedia-w:before{content:"\f266"}.fa-safari:before{content:"\f267"}.fa-chrome:before{content:"\f268"}.fa-firefox:before{content:"\f269"}.fa-opera:before{content:"\f26a"}.fa-internet-explorer:before{content:"\f26b"}.fa-tv:before,.fa-television:before{content:"\f26c"}.fa-contao:before{content:"\f26d"}.fa-500px:before{content:"\f26e"}.fa-amazon:before{content:"\f270"}.fa-calendar-plus-o:before{content:"\f271"}.fa-calendar-minus-o:before{content:"\f272"}.fa-calendar-times-o:before{content:"\f273"}.fa-calendar-check-o:before{content:"\f274"}.fa-industry:before{content:"\f275"}.fa-map-pin:before{content:"\f276"}.fa-map-signs:before{content:"\f277"}.fa-map-o:before{content:"\f278"}.fa-map:before{content:"\f279"}.fa-commenting:before{content:"\f27a"}.fa-commenting-o:before{content:"\f27b"}.fa-houzz:before{content:"\f27c"}.fa-vimeo:before{content:"\f27d"}.fa-black-tie:before{content:"\f27e"}.fa-fonticons:before{content:"\f280"}.fa-reddit-alien:before{content:"\f281"}.fa-edge:before{content:"\f282"}.fa-credit-card-alt:before{content:"\f283"}.fa-codiepie:before{content:"\f284"}.fa-modx:before{content:"\f285"}.fa-fort-awesome:before{content:"\f286"}.fa-usb:before{content:"\f287"}.fa-product-hunt:before{content:"\f288"}.fa-mixcloud:before{content:"\f289"}.fa-scribd:before{content:"\f28a"}.fa-pause-circle:before{content:"\f28b"}.fa-pause-circle-o:before{content:"\f28c"}.fa-stop-circle:before{content:"\f28d"}.fa-stop-circle-o:before{content:"\f28e"}.fa-shopping-bag:before{content:"\f290"}.fa-shopping-basket:before{content:"\f291"}.fa-hashtag:before{content:"\f292"}.fa-bluetooth:before{content:"\f293"}.fa-bluetooth-b:before{content:"\f294"}.fa-percent:before{content:"\f295"}.fa-gitlab:before{content:"\f296"}.fa-wpbeginner:before{content:"\f297"}.fa-wpforms:before{content:"\f298"}.fa-envira:before{content:"\f299"}.fa-universal-access:before{content:"\f29a"}.fa-wheelchair-alt:before{content:"\f29b"}.fa-question-circle-o:before{content:"\f29c"}.fa-blind:before{content:"\f29d"}.fa-audio-description:before{content:"\f29e"}.fa-volume-control-phone:before{content:"\f2a0"}.fa-braille:before{content:"\f2a1"}.fa-assistive-listening-systems:before{content:"\f2a2"}.fa-asl-interpreting:before,.fa-american-sign-language-interpreting:before{content:"\f2a3"}.fa-deafness:before,.fa-hard-of-hearing:before,.fa-deaf:before{content:"\f2a4"}.fa-glide:before{content:"\f2a5"}.fa-glide-g:before{content:"\f2a6"}.fa-signing:before,.fa-sign-language:before{content:"\f2a7"}.fa-low-vision:before{content:"\f2a8"}.fa-viadeo:before{content:"\f2a9"}.fa-viadeo-square:before{content:"\f2aa"}.fa-snapchat:before{content:"\f2ab"}.fa-snapchat-ghost:before{content:"\f2ac"}.fa-snapchat-square:before{content:"\f2ad"}.fa-pied-piper:before{content:"\f2ae"}.fa-first-order:before{content:"\f2b0"}.fa-yoast:before{content:"\f2b1"}.fa-themeisle:before{content:"\f2b2"}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:"\f2b3"}.fa-fa:before,.fa-font-awesome:before{content:"\f2b4"}.fa-handshake-o:before{content:"\f2b5"}.fa-envelope-open:before{content:"\f2b6"}.fa-envelope-open-o:before{content:"\f2b7"}.fa-linode:before{content:"\f2b8"}.fa-address-book:before{content:"\f2b9"}.fa-address-book-o:before{content:"\f2ba"}.fa-vcard:before,.fa-address-card:before{content:"\f2bb"}.fa-vcard-o:before,.fa-address-card-o:before{content:"\f2bc"}.fa-user-circle:before{content:"\f2bd"}.fa-user-circle-o:before{content:"\f2be"}.fa-user-o:before{content:"\f2c0"}.fa-id-badge:before{content:"\f2c1"}.fa-drivers-license:before,.fa-id-card:before{content:"\f2c2"}.fa-drivers-license-o:before,.fa-id-card-o:before{content:"\f2c3"}.fa-quora:before{content:"\f2c4"}.fa-free-code-camp:before{content:"\f2c5"}.fa-telegram:before{content:"\f2c6"}.fa-thermometer-4:before,.fa-thermometer:before,.fa-thermometer-full:before{content:"\f2c7"}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:"\f2c8"}.fa-thermometer-2:before,.fa-thermometer-half:before{content:"\f2c9"}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:"\f2ca"}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:"\f2cb"}.fa-shower:before{content:"\f2cc"}.fa-bathtub:before,.fa-s15:before,.fa-bath:before{content:"\f2cd"}.fa-podcast:before{content:"\f2ce"}.fa-window-maximize:before{content:"\f2d0"}.fa-window-minimize:before{content:"\f2d1"}.fa-window-restore:before{content:"\f2d2"}.fa-times-rectangle:before,.fa-window-close:before{content:"\f2d3"}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:"\f2d4"}.fa-bandcamp:before{content:"\f2d5"}.fa-grav:before{content:"\f2d6"}.fa-etsy:before{content:"\f2d7"}.fa-imdb:before{content:"\f2d8"}.fa-ravelry:before{content:"\f2d9"}.fa-eercast:before{content:"\f2da"}.fa-microchip:before{content:"\f2db"}.fa-snowflake-o:before{content:"\f2dc"}.fa-superpowers:before{content:"\f2dd"}.fa-wpexplorer:before{content:"\f2de"}.fa-meetup:before{content:"\f2e0"}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto} diff --git a/frontend/public/index.html b/frontend/public/index.html index 944f4141..3b3ab33b 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -6,12 +6,14 @@ + diff --git a/frontend/src/components/Node/NodeInstallationMatrix.vue b/frontend/src/components/Node/NodeInstallationMatrix.vue index c08dcaeb..8c628914 100644 --- a/frontend/src/components/Node/NodeInstallationMatrix.vue +++ b/frontend/src/components/Node/NodeInstallationMatrix.vue @@ -1,9 +1,214 @@ diff --git a/frontend/src/components/Settings/GitSettings.vue b/frontend/src/components/Settings/GitSettings.vue index 4a8c791b..048f2df6 100644 --- a/frontend/src/components/Settings/GitSettings.vue +++ b/frontend/src/components/Settings/GitSettings.vue @@ -1,159 +1,237 @@ @@ -318,4 +441,33 @@ export default { text-align: right; margin-top: 10px; } + + .git-settings .log { + height: calc(100vh - 280px); + overflow: auto; + } + + .git-settings .log .commit { + border-top: 1px solid rgb(244, 244, 245); + padding: 10px 0; + } + + .git-settings .log .commit .row { + display: flex; + justify-content: space-between; + } + + .git-settings .log .el-timeline-item { + /*cursor: pointer;*/ + } + + .git-settings .log .commit .row .tags .el-tag { + margin-right: 5px; + } + + .git-settings .log .commit .row .actions { + right: 0; + bottom: 5px; + position: absolute; + } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index f02ca169..899d9d44 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -346,6 +346,15 @@ export default { 'Executables': '执行文件', 'Latest Version': '最新版本', 'Version': '版本', + 'Installed': '已安装', + 'Not Installed': '未安装', + 'Installing': '正在安装', + 'Install All': '安装全部', + 'Other language installing': '其他语言正在安装', + 'This language is not installed yet.': '语言还未安装', + 'Languages': '语言', + 'Dependencies': '依赖', + 'Install on All Nodes': '安装在所有节点', // 弹出框 'Notification': '提示', diff --git a/frontend/src/views/login/index.vue b/frontend/src/views/login/index.vue index ab6cd54f..04cf9c74 100644 --- a/frontend/src/views/login/index.vue +++ b/frontend/src/views/login/index.vue @@ -71,8 +71,12 @@
中文 + | English
+
+ {{$t('Documentation')}} +
{{$t('You are running on a mobile device, which is not optimized yet. Please try with a laptop or desktop.')}} @@ -449,16 +453,17 @@ const initCanvas = () => { .lang { margin-top: 20px; text-align: center; + color: #666; span { cursor: pointer; margin: 10px; - color: #666; font-size: 14px; } span.active { font-weight: 600; + text-decoration: underline; } span:hover { @@ -466,6 +471,18 @@ const initCanvas = () => { } } + .documentation { + margin-top: 20px; + text-align: center; + font-size: 14px; + color: #409eff; + font-weight: bolder; + + &:hover { + text-decoration: underline; + } + } + .mobile-warning { margin-top: 20px; } diff --git a/frontend/src/views/node/NodeList.vue b/frontend/src/views/node/NodeList.vue index a3320f8d..1b3d69f5 100644 --- a/frontend/src/views/node/NodeList.vue +++ b/frontend/src/views/node/NodeList.vue @@ -139,12 +139,12 @@
- - - + + + diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index 02038caf..2b137a79 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -22,7 +22,7 @@ - + diff --git a/k8s/crawlab-master.yaml b/k8s/crawlab-master.yaml index 758cc051..d8dff5be 100644 --- a/k8s/crawlab-master.yaml +++ b/k8s/crawlab-master.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: crawlab - namespace: crawlab + namespace: crawlab-master spec: ports: - port: 8080 @@ -13,13 +13,12 @@ spec: type: NodePort --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: crawlab-master - namespace: crawlab + namespace: crawlab-master spec: - strategy: - type: Recreate + serviceName: crawlab-master selector: matchLabels: app: crawlab-master @@ -39,10 +38,16 @@ spec: value: "mongo" - name: CRAWLAB_REDIS_ADDRESS value: "redis" - # - name: CRAWLAB_SERVER_LANG_NODE - # value: "Y" - # - name: CRAWLAB_SERVER_LANG_JAVA - # value: "Y" + - name: CRAWLAB_SETTING_ALLOWREGISTER + value: "Y" + - name: CRAWLAB_SERVER_LANG_NODE + value: "N" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "N" + - name: CRAWLAB_SERVER_LANG_DOTNET + value: "N" + - name: CRAWLAB_SERVER_REGISTER_TYPE + value: "hostname" ports: - containerPort: 8080 - name: crawlab + name: crawlab \ No newline at end of file diff --git a/k8s/crawlab-worker.yaml b/k8s/crawlab-worker.yaml index 2efcd556..4f6e9002 100644 --- a/k8s/crawlab-worker.yaml +++ b/k8s/crawlab-worker.yaml @@ -1,12 +1,11 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: crawlab-worker - namespace: crawlab + namespace: crawlab-master spec: - replicas: 4 - strategy: - type: Recreate + serviceName: crawlab-worker + replicas: 2 selector: matchLabels: app: crawlab-worker @@ -26,7 +25,11 @@ spec: value: "mongo" - name: CRAWLAB_REDIS_ADDRESS value: "redis" - # - name: CRAWLAB_SERVER_LANG_NODE - # value: "Y" - # - name: CRAWLAB_SERVER_LANG_JAVA - # value: "Y" + - name: CRAWLAB_SERVER_LANG_NODE + value: "Y" + - name: CRAWLAB_SERVER_LANG_JAVA + value: "Y" + - name: CRAWLAB_SERVER_LANG_DOTNET + value: "Y" + - name: CRAWLAB_SERVER_REGISTER_TYPE + value: "hostname"