updated spiders

2026-01-22 17:31:03 +01:00 · 2019-06-06 17:53:07 +08:00
parent 40743e2f0a
commit 61c421094f
1 changed files with 4 additions and 1 deletions
--- a/crawlab/routes/spiders.py
+++ b/crawlab/routes/spiders.py
@@ -509,7 +509,10 @@ class SpiderApi(BaseApi):
                   }, r.status_code

        # get html parse tree
-        sel = etree.HTML(r.content.decode('utf-8'))
+        try:
+            sel = etree.HTML(r.content.decode('utf-8'))
+        except Exception as err:
+            sel = etree.HTML(r.content)

        # remove unnecessary tags
        unnecessary_tags = [