diff --git a/bin/run_flower.py b/bin/run_flower.py index 9e506464..d3a6900c 100644 --- a/bin/run_flower.py +++ b/bin/run_flower.py @@ -1,6 +1,14 @@ -from config.celery import BROKER_URL +import os +import sys import subprocess +# make sure the working directory is in system path +file_dir = os.path.dirname(os.path.realpath(__file__)) +root_path = os.path.abspath(os.path.join(file_dir, '..')) +sys.path.append(root_path) + +from config.celery import BROKER_URL + if __name__ == '__main__': p = subprocess.Popen(['celery', 'flower', '-b', BROKER_URL], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in iter(p.stdout.readline, 'b'): diff --git a/routes/spiders.py b/routes/spiders.py index faa71bed..43ef6ae9 100644 --- a/routes/spiders.py +++ b/routes/spiders.py @@ -120,6 +120,7 @@ class SpiderApi(BaseApi): dst = os.path.join(PROJECT_DEPLOY_FILE_FOLDER, str(spider.get('_id')), str(latest_version + 1)) # copy files + # TODO: multi-node copy files try: shutil.copytree(src=src, dst=dst) return { diff --git a/utils/deploy.py b/utils/deploy.py new file mode 100644 index 00000000..4cdae09b --- /dev/null +++ b/utils/deploy.py @@ -0,0 +1,13 @@ +import os, zipfile + + +# 打包目录为zip文件(未压缩) +def make_zip(source_dir, output_filename): + zipf = zipfile.ZipFile(output_filename, 'w') + pre_len = len(os.path.dirname(source_dir)) + for parent, dirnames, filenames in os.walk(source_dir): + for filename in filenames: + pathfile = os.path.join(parent, filename) + arcname = pathfile[pre_len:].strip(os.path.sep) # 相对路径 + zipf.write(pathfile, arcname) + zipf.close() diff --git a/utils/spider.py b/utils/spider.py index 229571a0..10b89720 100644 --- a/utils/spider.py +++ b/utils/spider.py @@ -1,3 +1,5 @@ +import os + from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE, SpiderType @@ -16,4 +18,6 @@ def get_lang_by_stats(stats: dict) -> LangType: def get_spider_type(path: str) -> SpiderType: - return SpiderType.SCRAPY + for file_name in os.listdir(path): + if file_name == 'scrapy.cfg': + return SpiderType.SCRAPY