From 8b8db8e94ee9e412957a567ae1d5beee53f9e7db Mon Sep 17 00:00:00 2001
From: Marvin Zhang <tikazyq@gmail.com>
Date: Tue, 26 Feb 2019 12:42:12 +0800
Subject: [PATCH] updated requirements.txt

---
 Dockerfile        | 2 ++
 requirements.txt  | 3 ---
 routes/nodes.py   | 4 ++--
 routes/spiders.py | 2 ++
 utils/spider.py   | 6 +++++-
 5 files changed, 11 insertions(+), 6 deletions(-)
 create mode 100644 Dockerfile

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..d567fa9d
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,2 @@
+FROM ubuntu:latest
+
diff --git a/requirements.txt b/requirements.txt
index f99bbc88..b7039e85 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,8 +13,6 @@ Click==7.0
 constantly==15.1.0
 cryptography==2.5
 cssselect==1.0.3
-Django==2.1.7
-django-cors-headers==2.4.0
 dnspython==1.16.0
 docopt==0.6.2
 eventlet==0.24.1
@@ -22,7 +20,6 @@ Flask==1.0.2
 Flask-Cors==3.0.7
 Flask-RESTful==0.3.7
 flower==0.9.2
-gerapy==0.8.5
 greenlet==0.4.15
 hyperlink==18.0.0
 idna==2.8
diff --git a/routes/nodes.py b/routes/nodes.py
index 3eb84a26..f9f04dc5 100644
--- a/routes/nodes.py
+++ b/routes/nodes.py
@@ -87,7 +87,7 @@ class NodeApi(BaseApi):
         items = db_manager.list('spiders')
 
     def get_deploys(self, id):
-        items = db_manager.list('deploys', {'node_id': id}, limit=10)
+        items = db_manager.list('deploys', {'node_id': id}, limit=10, sort_key='finish_ts')
         deploys = []
         for item in items:
             spider_id = item['spider_id']
@@ -100,7 +100,7 @@ class NodeApi(BaseApi):
         })
 
     def get_tasks(self, id):
-        items = db_manager.list('tasks', {'node_id': id}, limit=10)
+        items = db_manager.list('tasks', {'node_id': id}, limit=10, sort_key='create_ts')
         for item in items:
             spider_id = item['spider_id']
             spider = db_manager.get('spiders', id=str(spider_id))
diff --git a/routes/spiders.py b/routes/spiders.py
index 441e4209..faa71bed 100644
--- a/routes/spiders.py
+++ b/routes/spiders.py
@@ -71,6 +71,8 @@ class SpiderApi(BaseApi):
                     })
 
             items = db_manager.list('spiders', {})
+            for item in items:
+                item['latest_version'] = db_manager.get_latest_version(item['_id'])
 
         return jsonify({
             'status': 'ok',
diff --git a/utils/spider.py b/utils/spider.py
index d17a1d3a..229571a0 100644
--- a/utils/spider.py
+++ b/utils/spider.py
@@ -1,4 +1,4 @@
-from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE
+from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE, SpiderType
 
 
 def get_lang_by_stats(stats: dict) -> LangType:
@@ -13,3 +13,7 @@ def get_lang_by_stats(stats: dict) -> LangType:
     if FILE_SUFFIX_LANG_MAPPING.get(top_suffix) is not None:
         return FILE_SUFFIX_LANG_MAPPING.get(top_suffix)
     return LangType.OTHER
+
+
+def get_spider_type(path: str) -> SpiderType:
+    return SpiderType.SCRAPY