Version 20220729.05. Fix aborting item on bad status code on url: item. Keep old retry code otherwise.

pipeline.py

@@ -61,7 +61,7 @@ if not WGET_AT:
 #
 # Update this each time you make a non-cosmetic change.
 # It will be added to the WARC files and reported to the tracker.
-VERSION = '20220729.04'
+VERSION = '20220729.05'
 TRACKER_ID = 'reddit'
 TRACKER_HOST = 'legacy-api.arpa.li'
 MULTI_ITEM_SIZE = 20

reddit.lua

@@ -623,17 +623,34 @@ wget.callbacks.httploop_result = function(url, err, http_stat)
     return wget.actions.EXIT
   end
 
-  if status_code == 403 and string.match(url["url"], "^https?://v%.redd%.it/")
+  --[[if status_code == 403 and string.match(url["url"], "^https?://v%.redd%.it/")
     and selftext == "[deleted]" then
     return wget.actions.EXIT
-  end
+  end]]
   
   if status_code >= 500
-      or (status_code >= 400 and status_code ~= 404)
-      or status_code  == 0 then
+    or (status_code >= 400 and status_code ~= 404)
+    or status_code  == 0 then
+    if item_type == "url" then
+      abort_item()
+      return wget.actions.EXIT
+    end
     io.stdout:write("Server returned " .. http_stat.statcode .. " (" .. err .. "). Sleeping.\n")
     io.stdout:flush()
-    abort_item()
+    local maxtries = 8
+    if not allowed(url["url"]) then
+        maxtries = 0
+    end
+    if tries >= maxtries then
+      io.stdout:write("\nI give up...\n")
+      io.stdout:flush()
+      tries = 0
+      if allowed(url["url"]) then
+        return wget.actions.ABORT
+      else
+        return wget.actions.EXIT
+      end
+    end
     os.execute("sleep " .. math.floor(math.pow(2, tries)))
     tries = tries + 1
     return wget.actions.CONTINUE