git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

updated configurable spider

Browse Source
This commit is contained in:
Marvin Zhang
2019-05-26 17:10:04 +08:00
parent 15f9b7ef05
commit 5daebcb39b
14 changed files with 371 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
crawlab/routes/spiders.py
View File

@@ -19,7 +19,7 @@ from constants.task import TaskStatus
from db.manager import db_manager
from routes.base import BaseApi
from tasks.scheduler import scheduler
from tasks.spider import execute_spider
from tasks.spider import execute_spider, execute_config_spider
from utils import jsonify
from utils.deploy import zip_file, unzip_file
from utils.file import get_file_suffix_stats, get_file_suffix
@@ -83,8 +83,17 @@ class SpiderApi(BaseApi):
# spider item selector
('item_selector', str),
# spider item selector type
('item_selector_type', str),
# spider pagination selector
('pagination_selector', str),
# spider pagination selector type
('pagination_selector_type', str),
# whether to obey robots.txt
('obey_robots_txt', str),
)
def get(self, id=None, action=None):
@@ -251,7 +260,16 @@ class SpiderApi(BaseApi):
spider = db_manager.get('spiders', id=ObjectId(id))
job = execute_spider.delay(id, params)
# determine execute function
if spider['type'] == SpiderType.CONFIGURABLE:
# configurable spider
exec_func = execute_config_spider
else:
# customized spider
exec_func = execute_spider
# trigger an asynchronous job
job = exec_func.delay(id, params)
# create a new task
db_manager.save('tasks', {

14
crawlab/spiders/spiders/db.py Normal file
View File

@@ -0,0 +1,14 @@
import os
from pymongo import MongoClient
MONGO_HOST = os.environ.get('MONGO_HOST')
MONGO_PORT = int(os.environ.get('MONGO_PORT'))
MONGO_DB = os.environ.get('MONGO_DB')
mongo = MongoClient(host=MONGO_HOST,
port=MONGO_PORT)
db = mongo[MONGO_DB]
task_id = os.environ.get('CRAWLAB_TASK_ID')
col_name = os.environ.get('CRAWLAB_COLLECTION')
task = db['tasks'].find_one({'_id': task_id})
spider = db['spiders'].find_one({'_id': task['spider_id']})

8
crawlab/spiders/spiders/items.py
View File

@@ -7,8 +7,10 @@
import scrapy
from spiders.db import spider
class SpidersItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
fields = {f['name']: scrapy.Field() for f in spider['fields']}
fields['_id'] = scrapy.Field()
fields['task_id'] = scrapy.Field()

6
crawlab/spiders/spiders/pipelines.py
View File

@@ -4,8 +4,14 @@
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from spiders.db import db, col_name, task_id
class SpidersPipeline(object):
col = db[col_name]
def process_item(self, item, spider):
item['task_id'] = task_id
self.col.save(item)
return item

62
crawlab/spiders/spiders/settings.py
View File

@@ -8,83 +8,83 @@
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from spiders.db import spider
BOT_NAME = 'spiders'
BOT_NAME = 'Crawlab Spider'
SPIDER_MODULES = ['spiders.spiders']
NEWSPIDER_MODULE = 'spiders.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'spiders (+http://www.yourdomain.com)'
# USER_AGENT = 'spiders (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
ROBOTSTXT_OBEY = spider.get('obey_robots_txt') or True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# }
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# SPIDER_MIDDLEWARES = {
# 'spiders.middlewares.SpidersSpiderMiddleware': 543,
#}
# }
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# DOWNLOADER_MIDDLEWARES = {
# 'spiders.middlewares.SpidersDownloaderMiddleware': 543,
#}
# }
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# }
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'spiders.pipelines.SpidersPipeline': 300,
#}
ITEM_PIPELINES = {
'spiders.pipelines.SpidersPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

63
crawlab/spiders/spiders/spiders/config_spider.py Normal file
View File

@@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
from urllib.parse import urlparse
import scrapy
from spiders.db import spider
from spiders.items import SpidersItem
class NormalSpiderSpider(scrapy.Spider):
name = 'config_spider'
# allowed_domains = []
start_urls = [spider['start_url']]
def parse(self, response):
if spider['item_selector_type'] == 'xpath':
# xpath selector
items = response.xpath(spider['item_selector'])
else:
# css selector
items = response.css(spider['item_selector'])
for _item in items:
item = SpidersItem()
for f in spider['fields']:
if f['type'] == 'xpath':
# xpath selector
if f['extract_type'] == 'text':
# text content
query = f['query'] + '/text()'
else:
# attribute
attribute = f["attribute"]
query = f['query'] + f'/@("{attribute}")'
item[f['name']] = _item.xpath(query).extract_first()
else:
# css selector
if f['extract_type'] == 'text':
# text content
query = f['query'] + '::text'
else:
# attribute
attribute = f["attribute"]
query = f['query'] + f'::attr("{attribute}")'
item[f['name']] = _item.css(query).extract_first()
yield item
# pagination
if spider.get('pagination_selector') is not None:
if spider['pagination_selector_type'] == 'xpath':
# xpath selector
next_url = response.xpath(spider['pagination_selector'] + '/@href').extract_first()
else:
# css selector
next_url = response.css(spider['pagination_selector'] + '::attr("href")').extract_first()
# found next url
if next_url is not None:
if not next_url.startswith('http') and not next_url.startswith('//'):
u = urlparse(response.url)
next_url = f'{u.scheme}://{u.netloc}{next_url}'
yield scrapy.Request(url=next_url)

123
crawlab/tasks/spider.py
View File

@@ -6,13 +6,15 @@ from time import sleep
from bson import ObjectId
from pymongo import ASCENDING, DESCENDING
from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, PYTHON_ENV_PATH
from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, PYTHON_ENV_PATH, MONGO_HOST, MONGO_PORT, MONGO_DB
from constants.task import TaskStatus
from db.manager import db_manager
from .celery import celery_app
import subprocess
from utils.log import other as logger
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
def get_task(id: str):
i = 0
@@ -112,6 +114,125 @@ def execute_spider(self, id: str, params: str = None):
env=env,
bufsize=1)
# update pid
db_manager.update_one(col_name='tasks', id=task_id, values={
'pid': p.pid
})
# get output from the process
_stdout, _stderr = p.communicate()
# get return code
code = p.poll()
if code == 0:
status = TaskStatus.SUCCESS
else:
status = TaskStatus.FAILURE
except Exception as err:
logger.error(err)
stderr.write(str(err))
status = TaskStatus.FAILURE
# save task when the task is finished
finish_ts = datetime.utcnow()
db_manager.update_one('tasks', id=task_id, values={
'finish_ts': finish_ts,
'duration': (finish_ts - task['create_ts']).total_seconds(),
'status': status
})
task = db_manager.get('tasks', id=id)
# close log file streams
stdout.flush()
stderr.flush()
stdout.close()
stderr.close()
return task
@celery_app.task(bind=True)
def execute_config_spider(self, id: str, params: str = None):
task_id = self.request.id
hostname = self.request.hostname
spider = db_manager.get('spiders', id=id)
# get task object and return if not found
task = get_task(task_id)
if task is None:
return
# current working directory
current_working_directory = os.path.join(BASE_DIR, 'spiders')
# log info
logger.info('task_id: %s' % task_id)
logger.info('hostname: %s' % hostname)
logger.info('current_working_directory: %s' % current_working_directory)
logger.info('spider_id: %s' % id)
# make sure the log folder exists
log_path = os.path.join(PROJECT_LOGS_FOLDER, id)
if not os.path.exists(log_path):
os.makedirs(log_path)
# open log file streams
log_file_path = os.path.join(log_path, '%s.log' % datetime.now().strftime('%Y%m%d%H%M%S'))
stdout = open(log_file_path, 'a')
stderr = open(log_file_path, 'a')
# update task status as started
db_manager.update_one('tasks', id=task_id, values={
'start_ts': datetime.utcnow(),
'node_id': hostname,
'hostname': hostname,
'log_file_path': log_file_path,
'status': TaskStatus.STARTED
})
# pass params as env variables
env = os.environ.copy()
# custom environment variables
if spider.get('envs'):
for _env in spider.get('envs'):
env[_env['name']] = _env['value']
# task id environment variable
env['CRAWLAB_TASK_ID'] = task_id
# collection environment variable
if spider.get('col'):
env['CRAWLAB_COLLECTION'] = spider.get('col')
# create index to speed results data retrieval
db_manager.create_index(spider.get('col'), [('task_id', ASCENDING)])
# mongodb environment variables
env['MONGO_HOST'] = MONGO_HOST
env['MONGO_PORT'] = str(MONGO_PORT)
env['MONGO_DB'] = MONGO_DB
cmd_arr = [
sys.executable,
'-m',
'scrapy',
'crawl',
'config_spider'
]
try:
p = subprocess.Popen(cmd_arr,
stdout=stdout.fileno(),
stderr=stderr.fileno(),
cwd=current_working_directory,
env=env,
bufsize=1)
# update pid
db_manager.update_one(col_name='tasks', id=task_id, values={
'pid': p.pid
})
# get output from the process
_stdout, _stderr = p.communicate()

65
frontend/src/components/Config/ConfigList.vue
View File

@@ -20,9 +20,9 @@
</el-dialog>
<!--./preview results-->
<el-row style="margin-top: 10px;">
<el-row>
<el-col :span="11" offset="1">
<el-form label-width="100px">
<el-form label-width="150px">
<el-form-item :label="$t('Crawl Type')">
<el-button-group>
<el-button v-for="type in crawlTypeList"
@@ -36,28 +36,54 @@
<el-form-item :label="$t('Start URL')">
<el-input v-model="spiderForm.start_url" :placeholder="$t('Start URL')"></el-input>
</el-form-item>
<el-form-item :label="$t('Obey robots.txt')">
<el-switch v-model="spiderForm.obey_robots_txt" :placeholder="$t('Obey robots.txt')"></el-switch>
</el-form-item>
</el-form>
</el-col>
<el-col :span="11" :offset="1">
<el-form label-width="150px">
<el-form-item :label="$t('Item Selector')"
v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
<el-input v-model="spiderForm.item_selector" :placeholder="$t('Item Selector')"></el-input>
<el-select style="width: 35%;margin-right: 10px;"
v-model="spiderForm.item_selector_type"
:placeholder="$t('Item Selector Type')">
<el-option value="xpath" :label="$t('XPath')"></el-option>
<el-option value="css" :label="$t('CSS')"></el-option>
</el-select>
<el-input style="width: calc(65% - 10px);"
v-model="spiderForm.item_selector"
:placeholder="$t('Item Selector')">
</el-input>
</el-form-item>
<el-form-item :label="$t('Pagination Selector')"
v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
<el-input v-model="spiderForm.pagination_selector" :placeholder="$t('Pagination Selector')"></el-input>
<el-select style="width: 35%;margin-right: 10px;"
v-model="spiderForm.pagination_selector_type"
:placeholder="$t('Pagination Selector Type')">
<el-option value="xpath" :label="$t('XPath')"></el-option>
<el-option value="css" :label="$t('CSS')"></el-option>
</el-select>
<el-input style="width: calc(65% - 10px);"
v-model="spiderForm.pagination_selector"
:placeholder="$t('Pagination Selector')">
</el-input>
</el-form-item>
</el-form>
</el-col>
</el-row>
<!--button group-->
<el-row>
<div class="button-group">
<el-button type="primary" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>
<el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
<el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
<el-row style="margin-top: 10px">
<div class="button-group-wrapper">
<div class="button-group">
<el-button type="primary" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>
</div>
<div class="button-group">
<el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
<el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
<el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
</div>
</div>
</el-row>
<!--./button group-->
@@ -200,6 +226,18 @@ export default {
this.previewLoading = false
})
})
},
onCrawl () {
this.$confirm(this.$t('Are you sure to run this spider?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel')
})
.then(() => {
this.$store.dispatch('spider/crawlSpider', this.spiderForm._id)
.then(() => {
this.$message.success(this.$t(`Spider task has been scheduled`))
})
})
}
},
created () {
@@ -215,13 +253,15 @@ export default {
}
if (!this.spiderForm.crawl_type) this.$set(this.spiderForm, 'crawl_type', 'list')
if (!this.spiderForm.start_url) this.$set(this.spiderForm, 'start_url', 'http://example.com')
if (!this.spiderForm.item_selector_type) this.$set(this.spiderForm, 'item_selector_type', 'css')
if (!this.spiderForm.pagination_selector_type) this.$set(this.spiderForm, 'pagination_selector_type', 'css')
if (!this.spiderForm.obey_robots_txt) this.$set(this.spiderForm, 'obey_robots_txt', true)
}
}
</script>
<style scoped>
.el-table {
margin-top: 10px;
}
.el-table.edit >>> .el-table__body td {
@@ -248,6 +288,11 @@ export default {
line-height: 36px;
}
.button-group-wrapper {
display: flex;
justify-content: space-between;
}
.button-group {
text-align: right;
}

7
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -99,13 +99,14 @@ export default {
]),
isShowRun () {
if (this.isCustomized) {
// customized spider
if (!this.spiderForm.deploy_ts) {
return false
}
return !!this.spiderForm.cmd
} else {
// TODO: has to add rules
return false
// configurable spider
return !!this.spiderForm.fields
}
},
isCustomized () {
@@ -113,7 +114,7 @@ export default {
}
},
methods: {
onRun () {
onCrawl () {
const row = this.spiderForm
this.$refs['spiderForm'].validate(res => {
if (res) {

16
frontend/src/i18n/zh.js
View File

@@ -112,6 +112,22 @@ export default {
'Customized': '自定义',
'Text': '文本',
'Attribute': '属性',
'Field Name': '字段名称',
'Query Type': '查询类别',
'Query': '查询',
'Extract Type': '提取类别',
'CSS Selector': 'CSS选择器',
'Crawl Type': '抓取类别',
'List Only': '仅列表',
'Detail Only': '仅详情页',
'List + Detail': '列表＋详情页',
'Start URL': '开始URL',
'Item Selector': '列表项选择器',
'Item Selector Type': '列表项选择器类别',
'Pagination Selector': '分页选择器',
'Pagination Selector Type': '分页项选择器类别',
'Preview Results': '预览结果',
'Obey robots.txt': '遵守Robots协议',
// 爬虫列表
'Name': '名称',

5
frontend/src/store/modules/spider.js
View File

@@ -106,7 +106,10 @@ const actions = {
crawl_type: state.spiderForm.crawl_type,
start_url: state.spiderForm.start_url,
item_selector: state.spiderForm.item_selector,
pagination_selector: state.spiderForm.pagination_selector
item_selector_type: state.spiderForm.item_selector_type,
pagination_selector: state.spiderForm.pagination_selector,
pagination_selector_type: state.spiderForm.pagination_selector_type,
obey_robots_txt: state.spiderForm.obey_robots_txt
})
.then(() => {
dispatch('getSpiderList')

2
frontend/src/views/schedule/ScheduleList.vue
View File

@@ -196,7 +196,7 @@ export default {
}, 100)
})
},
onRun () {
onCrawl () {
}
},
created () {

18
frontend/src/views/spider/SpiderList.vue
View File

@@ -426,13 +426,19 @@ export default {
})
},
isShowRun (row) {
if (!row.deploy_ts) {
return false
if (this.isCustomized(row)) {
// customized spider
if (!row.deploy_ts) {
return false
}
return !!row.cmd
} else {
// configurable spider
return !!row.fields
}
if (!row.cmd) {
return false
}
return true
},
isCustomized (row) {
return row.type === 'customized'
},
fetchSiteSuggestions (keyword, callback) {
this.$request.get('/sites', {

23
frontend/src/views/task/TaskDetail.vue
View File

@@ -6,11 +6,13 @@
<task-overview/>
</el-tab-pane>
<el-tab-pane :label="$t('Log')" name="log">
<div class="log-view">
<pre>
{{taskLog}}
</pre>
</div>
<el-card>
<div class="log-view">
<pre>
{{taskLog}}
</pre>
</div>
</el-card>
</el-tab-pane>
<el-tab-pane :label="$t('Results')" name="results">
<general-table-view :data="taskResultsData"
@@ -101,4 +103,15 @@ export default {
.selector .el-select {
padding-left: 10px;
}
.log-view {
margin: 20px;
height: 640px;
}
.log-view pre {
height: 100%;
overflow-x: auto;
overflow-y: auto;
}
</style>