prepare for multi-node deployment

bin/run_flower.py

@@ -1,6 +1,14 @@
-from config.celery import BROKER_URL
+import os
+import sys
 import subprocess
 
+# make sure the working directory is in system path
+file_dir = os.path.dirname(os.path.realpath(__file__))
+root_path = os.path.abspath(os.path.join(file_dir, '..'))
+sys.path.append(root_path)
+
+from config.celery import BROKER_URL
+
 if __name__ == '__main__':
     p = subprocess.Popen(['celery', 'flower', '-b', BROKER_URL], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     for line in iter(p.stdout.readline, 'b'):

routes/spiders.py

@@ -120,6 +120,7 @@ class SpiderApi(BaseApi):
         dst = os.path.join(PROJECT_DEPLOY_FILE_FOLDER, str(spider.get('_id')), str(latest_version + 1))
 
         # copy files
+        # TODO: multi-node copy files
         try:
             shutil.copytree(src=src, dst=dst)
             return {

utils/deploy.py

@@ -0,0 +1,13 @@
+import os, zipfile
+
+
+# 打包目录为zip文件（未压缩）
+def make_zip(source_dir, output_filename):
+    zipf = zipfile.ZipFile(output_filename, 'w')
+    pre_len = len(os.path.dirname(source_dir))
+    for parent, dirnames, filenames in os.walk(source_dir):
+        for filename in filenames:
+            pathfile = os.path.join(parent, filename)
+            arcname = pathfile[pre_len:].strip(os.path.sep)  # 相对路径
+            zipf.write(pathfile, arcname)
+    zipf.close()

utils/spider.py

@@ -1,3 +1,5 @@
+import os
+
 from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE, SpiderType
 
 
@@ -16,4 +18,6 @@ def get_lang_by_stats(stats: dict) -> LangType:
 
 
 def get_spider_type(path: str) -> SpiderType:
-    return SpiderType.SCRAPY
+    for file_name in os.listdir(path):
+        if file_name == 'scrapy.cfg':
+            return SpiderType.SCRAPY