updated requirements.txt

Dockerfile

@@ -0,0 +1,2 @@
+FROM ubuntu:latest
+

requirements.txt

@@ -13,8 +13,6 @@ Click==7.0
 constantly==15.1.0
 cryptography==2.5
 cssselect==1.0.3
-Django==2.1.7
-django-cors-headers==2.4.0
 dnspython==1.16.0
 docopt==0.6.2
 eventlet==0.24.1
@@ -22,7 +20,6 @@ Flask==1.0.2
 Flask-Cors==3.0.7
 Flask-RESTful==0.3.7
 flower==0.9.2
-gerapy==0.8.5
 greenlet==0.4.15
 hyperlink==18.0.0
 idna==2.8

routes/nodes.py

@@ -87,7 +87,7 @@ class NodeApi(BaseApi):
         items = db_manager.list('spiders')
 
     def get_deploys(self, id):
-        items = db_manager.list('deploys', {'node_id': id}, limit=10)
+        items = db_manager.list('deploys', {'node_id': id}, limit=10, sort_key='finish_ts')
         deploys = []
         for item in items:
             spider_id = item['spider_id']
@@ -100,7 +100,7 @@ class NodeApi(BaseApi):
         })
 
     def get_tasks(self, id):
-        items = db_manager.list('tasks', {'node_id': id}, limit=10)
+        items = db_manager.list('tasks', {'node_id': id}, limit=10, sort_key='create_ts')
         for item in items:
             spider_id = item['spider_id']
             spider = db_manager.get('spiders', id=str(spider_id))

routes/spiders.py

@@ -71,6 +71,8 @@ class SpiderApi(BaseApi):
                     })
 
             items = db_manager.list('spiders', {})
+            for item in items:
+                item['latest_version'] = db_manager.get_latest_version(item['_id'])
 
         return jsonify({
             'status': 'ok',

utils/spider.py

@@ -1,4 +1,4 @@
-from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE
+from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE, SpiderType
 
 
 def get_lang_by_stats(stats: dict) -> LangType:
@@ -13,3 +13,7 @@ def get_lang_by_stats(stats: dict) -> LangType:
     if FILE_SUFFIX_LANG_MAPPING.get(top_suffix) is not None:
         return FILE_SUFFIX_LANG_MAPPING.get(top_suffix)
     return LangType.OTHER
+
+
+def get_spider_type(path: str) -> SpiderType:
+    return SpiderType.SCRAPY