git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

added chromedriver

Browse Source
This commit is contained in:
marvzhang
2020-07-30 12:02:11 +08:00
parent 27fd16f345
commit 596a2a7e4d
6 changed files with 157 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
backend/constants/system.go
View File

@@ -18,3 +18,8 @@ const (
InstallStatusInstallingOther = "installing-other"
InstallStatusInstalled = "installed"
)
const (
LangTypeLang = "lang"
LangTypeWebDriver = "webdriver"
)

1
backend/entity/system.go
View File

@@ -24,6 +24,7 @@ type Lang struct {
InstallStatus string `json:"install_status"`
DepFileName string `json:"dep_file_name"`
InstallDepArgs string `json:"install_dep_cmd"`
Type string `json:"type"`
}
type Dependency struct {

18
backend/scripts/install-chromedriver.sh Normal file
View File

@@ -0,0 +1,18 @@
export DEBIAN_FRONTEND=noninteractive \
&& apt-get update \
&& apt-get install \
unzip \
&& \
DL=https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \
&& curl -sL "$DL" > /tmp/chrome.deb \
&& apt install --no-install-recommends --no-install-suggests -y \
/tmp/chrome.deb \
&& CHROMIUM_FLAGS='--no-sandbox --disable-dev-shm-usage' \
# Patch Chrome launch script and append CHROMIUM_FLAGS to the last line:
&& sed -i '${s/$/'" $CHROMIUM_FLAGS"'/}' /opt/google/chrome/google-chrome \
&& BASE_URL=https://chromedriver.storage.googleapis.com \
&& VERSION=$(curl -sL "$BASE_URL/LATEST_RELEASE") \
&& curl -sL "$BASE_URL/$VERSION/chromedriver_linux64.zip" -o /tmp/driver.zip \
&& unzip /tmp/driver.zip \
&& chmod 755 chromedriver \
&& mv chromedriver /usr/local/bin

1
backend/services/system.go
View File

@@ -71,6 +71,7 @@ func GetLangList(nodeId string) []entity.Lang {
return list
}
// 获取语言安装状态
func GetLangInstallStatus(nodeId string, lang entity.Lang) (string, error) {
if IsMasterNode(nodeId) {
lang := rpc.GetLangLocal(lang)

17
backend/utils/system.go
View File

@@ -1,6 +1,7 @@
package utils
import (
"crawlab/constants"
"crawlab/entity"
"encoding/json"
"github.com/apex/log"
@@ -10,6 +11,7 @@ import (
func GetLangList() []entity.Lang {
list := []entity.Lang{
// 语言
{
Name: "Python",
ExecutableName: "python",
@@ -18,6 +20,7 @@ func GetLangList() []entity.Lang {
LockPath: "/tmp/install-python.lock",
DepFileName: "requirements.txt",
InstallDepArgs: "install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt",
Type: constants.LangTypeLang,
},
{
Name: "Node.js",
@@ -28,6 +31,7 @@ func GetLangList() []entity.Lang {
InstallScript: "install-nodejs.sh",
DepFileName: "package.json",
InstallDepArgs: "install -g --registry=https://registry.npm.taobao.org",
Type: constants.LangTypeLang,
},
{
Name: "Java",
@@ -35,6 +39,7 @@ func GetLangList() []entity.Lang {
ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"},
LockPath: "/tmp/install-java.lock",
InstallScript: "install-java.sh",
Type: constants.LangTypeLang,
},
{
Name: ".Net Core",
@@ -42,6 +47,7 @@ func GetLangList() []entity.Lang {
ExecutablePaths: []string{"/usr/bin/dotnet", "/usr/local/bin/dotnet"},
LockPath: "/tmp/install-dotnet.lock",
InstallScript: "install-dotnet.sh",
Type: constants.LangTypeLang,
},
{
Name: "PHP",
@@ -49,6 +55,7 @@ func GetLangList() []entity.Lang {
ExecutablePaths: []string{"/usr/bin/php", "/usr/local/bin/php"},
LockPath: "/tmp/install-php.lock",
InstallScript: "install-php.sh",
Type: constants.LangTypeLang,
},
{
Name: "Golang",
@@ -56,6 +63,16 @@ func GetLangList() []entity.Lang {
ExecutablePaths: []string{"/usr/bin/go", "/usr/local/bin/go"},
LockPath: "/tmp/install-go.lock",
InstallScript: "install-go.sh",
Type: constants.LangTypeLang,
},
// WebDriver
{
Name: "Chrome Driver",
ExecutableName: "chromedriver",
ExecutablePaths: []string{"/usr/bin/chromedriver", "/usr/local/bin/chromedriver"},
LockPath: "/tmp/install-chromedriver.lock",
InstallScript: "install-chromedriver.sh",
Type: constants.LangTypeWebDriver,
},
}
return list

126
frontend/src/components/Node/NodeInstallationMatrix.vue
View File

@@ -72,7 +72,11 @@
<i class="el-icon-error" />
{{ $t('Not Installed') }}
</el-tag>
<el-button type="primary" size="mini" @click="onInstallLang(scope.row._id, scope.column.label, $event)">
<el-button
type="primary"
size="mini"
@click="onInstallLang(scope.row._id, scope.column.label, $event)"
>
{{ $t('Install') }}
</el-button>
</div>
@@ -203,6 +207,97 @@
</el-table-column>
</el-table>
</el-tab-pane>
<el-tab-pane label="Web Driver" name="webdriver">
<div class="webdriver-table">
<el-table
class="table"
:data="nodeList"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white',height:'50px'}"
border
@row-click="onLangTableRowClick"
>
<el-table-column
:label="$t('Node')"
width="240px"
prop="name"
fixed
/>
<el-table-column
:label="$t('nodeList.type')"
width="120px"
fixed
>
<template slot-scope="scope">
<el-tag v-if="scope.row.is_master" type="primary">{{ $t('Master') }}</el-tag>
<el-tag v-else type="warning">{{ $t('Worker') }}</el-tag>
</template>
</el-table-column>
<el-table-column
:label="$t('Status')"
width="120px"
fixed
>
<template slot-scope="scope">
<el-tag v-if="scope.row.status === 'offline'" type="info">{{ $t('Offline') }}</el-tag>
<el-tag v-else-if="scope.row.status === 'online'" type="success">{{ $t('Online') }}</el-tag>
<el-tag v-else type="danger">{{ $t('Unavailable') }}</el-tag>
</template>
</el-table-column>
<el-table-column
v-for="wd in webdrivers"
:key="wd.name"
:label="wd.label"
width="220px"
>
<template slot="header" slot-scope="scope">
<div class="header-with-action">
<span>{{ scope.column.label }}</span>
<el-button type="primary" size="mini" @click="onInstallLangAll(scope.column.label, $event)">
{{ $t('Install') }}
</el-button>
</div>
</template>
<template slot-scope="scope">
<template v-if="getLangInstallStatus(scope.row._id, wd.name) === 'installed'">
<el-tag type="success">
<i class="el-icon-check" />
{{ $t('Installed') }}
</el-tag>
</template>
<template v-else-if="getLangInstallStatus(scope.row._id, wd.name) === 'installing'">
<el-tag type="warning">
<i class="el-icon-loading" />
{{ $t('Installing') }}
</el-tag>
</template>
<template
v-else-if="['installing-other', 'not-installed'].includes(getLangInstallStatus(scope.row._id, wd.name))"
>
<div class="cell-with-action">
<el-tag type="danger">
<i class="el-icon-error" />
{{ $t('Not Installed') }}
</el-tag>
<el-button
type="primary"
size="mini"
@click="onInstallLang(scope.row._id, scope.column.label, $event)"
>
{{ $t('Install') }}
</el-button>
</div>
</template>
<template v-else-if="getLangInstallStatus(scope.row._id, wd.name) === 'na'">
<el-tag type="info">
<i class="el-icon-question" />
{{ $t('N/A') }}
</el-tag>
</template>
</template>
</el-table-column>
</el-table>
</div>
</el-tab-pane>
</el-tabs>
</div>
</template>
@@ -220,13 +315,16 @@
},
data() {
return {
langs: [
{ label: 'Python', name: 'python', hasDeps: true },
{ label: 'Node.js', name: 'node', hasDeps: true },
{ label: 'Java', name: 'java', hasDeps: false },
{ label: '.Net Core', name: 'dotnet', hasDeps: false },
{ label: 'PHP', name: 'php', hasDeps: false },
{ label: 'Golang', name: 'go', hasDeps: false }
allLangs: [
// 语言
{ label: 'Python', name: 'python', hasDeps: true, type: 'lang' },
{ label: 'Node.js', name: 'node', hasDeps: true, type: 'lang' },
{ label: 'Java', name: 'java', hasDeps: false, type: 'lang' },
{ label: '.Net Core', name: 'dotnet', hasDeps: false, type: 'lang' },
{ label: 'PHP', name: 'php', hasDeps: false, type: 'lang' },
{ label: 'Golang', name: 'go', hasDeps: false, type: 'lang' },
// web driver
{ label: 'Chrome Driver', name: 'chromedriver', type: 'webdriver' }
],
langsDataDict: {},
handle: undefined,
@@ -244,6 +342,12 @@
...mapState('node', [
'nodeList'
]),
langs() {
return this.allLangs.filter(d => d.type === 'lang')
},
webdrivers() {
return this.allLangs.filter(d => d.type === 'webdriver')
},
activeNodes() {
return this.nodeList.filter(d => d.status === 'online')
},
@@ -255,7 +359,7 @@
})
},
langsWithDeps() {
return this.langs.filter(l => l.hasDeps)
return this.allLangs.filter(l => l.hasDeps)
}
},
watch: {
@@ -316,8 +420,8 @@
return lang.install_status
},
getLangFromLabel(label) {
for (let i = 0; i < this.langs.length; i++) {
const lang = this.langs[i]
for (let i = 0; i < this.allLangs.length; i++) {
const lang = this.allLangs[i]
if (lang.label === label) {
return lang
}