updated configurable spider

crawlab/spiders/spiders/db.py

@@ -0,0 +1,14 @@
+import os
+
+from pymongo import MongoClient
+
+MONGO_HOST = os.environ.get('MONGO_HOST')
+MONGO_PORT = int(os.environ.get('MONGO_PORT'))
+MONGO_DB = os.environ.get('MONGO_DB')
+mongo = MongoClient(host=MONGO_HOST,
+                    port=MONGO_PORT)
+db = mongo[MONGO_DB]
+task_id = os.environ.get('CRAWLAB_TASK_ID')
+col_name = os.environ.get('CRAWLAB_COLLECTION')
+task = db['tasks'].find_one({'_id': task_id})
+spider = db['spiders'].find_one({'_id': task['spider_id']})

crawlab/spiders/spiders/items.py

@@ -7,8 +7,10 @@
 
 import scrapy
 
+from spiders.db import spider
+
 
 class SpidersItem(scrapy.Item):
-    # define the fields for your item here like:
-    # name = scrapy.Field()
-    pass
+    fields = {f['name']: scrapy.Field() for f in spider['fields']}
+    fields['_id'] = scrapy.Field()
+    fields['task_id'] = scrapy.Field()

crawlab/spiders/spiders/pipelines.py

@@ -4,8 +4,14 @@
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+from spiders.db import db, col_name, task_id
 
 
 class SpidersPipeline(object):
+    col = db[col_name]
+
     def process_item(self, item, spider):
+        item['task_id'] = task_id
+        self.col.save(item)
+
         return item

crawlab/spiders/spiders/settings.py

@@ -8,83 +8,83 @@
 #     https://doc.scrapy.org/en/latest/topics/settings.html
 #     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
 #     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+from spiders.db import spider
 
-BOT_NAME = 'spiders'
+BOT_NAME = 'Crawlab Spider'
 
 SPIDER_MODULES = ['spiders.spiders']
 NEWSPIDER_MODULE = 'spiders.spiders'
 
-
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = 'spiders (+http://www.yourdomain.com)'
+# USER_AGENT = 'spiders (+http://www.yourdomain.com)'
 
 # Obey robots.txt rules
-ROBOTSTXT_OBEY = True
+ROBOTSTXT_OBEY = spider.get('obey_robots_txt') or True
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS = 32
+# CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
 # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
+# DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
+# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+# CONCURRENT_REQUESTS_PER_IP = 16
 
 # Disable cookies (enabled by default)
-#COOKIES_ENABLED = False
+# COOKIES_ENABLED = False
 
 # Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED = False
+# TELNETCONSOLE_ENABLED = False
 
 # Override the default request headers:
-#DEFAULT_REQUEST_HEADERS = {
+# DEFAULT_REQUEST_HEADERS = {
 #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 #   'Accept-Language': 'en',
-#}
+# }
 
 # Enable or disable spider middlewares
 # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
-#SPIDER_MIDDLEWARES = {
+# SPIDER_MIDDLEWARES = {
 #    'spiders.middlewares.SpidersSpiderMiddleware': 543,
-#}
+# }
 
 # Enable or disable downloader middlewares
 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
+# DOWNLOADER_MIDDLEWARES = {
 #    'spiders.middlewares.SpidersDownloaderMiddleware': 543,
-#}
+# }
 
 # Enable or disable extensions
 # See https://doc.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
+# EXTENSIONS = {
 #    'scrapy.extensions.telnet.TelnetConsole': None,
-#}
+# }
 
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
-#    'spiders.pipelines.SpidersPipeline': 300,
-#}
+ITEM_PIPELINES = {
+    'spiders.pipelines.SpidersPipeline': 300,
+}
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
+# AUTOTHROTTLE_ENABLED = True
 # The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
+# AUTOTHROTTLE_START_DELAY = 5
 # The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
+# AUTOTHROTTLE_MAX_DELAY = 60
 # The average number of requests Scrapy should be sending in parallel to
 # each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 # Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
+# AUTOTHROTTLE_DEBUG = False
 
 # Enable and configure HTTP caching (disabled by default)
 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = 'httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+# HTTPCACHE_ENABLED = True
+# HTTPCACHE_EXPIRATION_SECS = 0
+# HTTPCACHE_DIR = 'httpcache'
+# HTTPCACHE_IGNORE_HTTP_CODES = []
+# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

crawlab/spiders/spiders/spiders/config_spider.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+from urllib.parse import urlparse
+
+import scrapy
+
+from spiders.db import spider
+from spiders.items import SpidersItem
+
+
+class NormalSpiderSpider(scrapy.Spider):
+    name = 'config_spider'
+    # allowed_domains = []
+    start_urls = [spider['start_url']]
+
+    def parse(self, response):
+        if spider['item_selector_type'] == 'xpath':
+            # xpath selector
+            items = response.xpath(spider['item_selector'])
+        else:
+            # css selector
+            items = response.css(spider['item_selector'])
+        for _item in items:
+            item = SpidersItem()
+            for f in spider['fields']:
+                if f['type'] == 'xpath':
+                    # xpath selector
+                    if f['extract_type'] == 'text':
+                        # text content
+                        query = f['query'] + '/text()'
+                    else:
+                        # attribute
+                        attribute = f["attribute"]
+                        query = f['query'] + f'/@("{attribute}")'
+                    item[f['name']] = _item.xpath(query).extract_first()
+
+                else:
+                    # css selector
+                    if f['extract_type'] == 'text':
+                        # text content
+                        query = f['query'] + '::text'
+                    else:
+                        # attribute
+                        attribute = f["attribute"]
+                        query = f['query'] + f'::attr("{attribute}")'
+                    item[f['name']] = _item.css(query).extract_first()
+
+                yield item
+
+        # pagination
+        if spider.get('pagination_selector') is not None:
+            if spider['pagination_selector_type'] == 'xpath':
+                # xpath selector
+                next_url = response.xpath(spider['pagination_selector'] + '/@href').extract_first()
+            else:
+                # css selector
+                next_url = response.css(spider['pagination_selector'] + '::attr("href")').extract_first()
+
+            # found next url
+            if next_url is not None:
+                if not next_url.startswith('http') and not next_url.startswith('//'):
+                    u = urlparse(response.url)
+                    next_url = f'{u.scheme}://{u.netloc}{next_url}'
+                yield scrapy.Request(url=next_url)