Finishing RFC 5005 archive fetcher (for #1109). Also fixing mongo com…

…mands and adding checks for archive subscribers.
samuelclay · Apr 29, 2022 · de1fb57 · de1fb57
1 parent f42b26b
commit de1fb57
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 10 deletions.
diff --git a/Makefile b/Makefile
@@ -194,11 +194,13 @@ oldfirewall:
 repairmongo:
 	- sudo docker run -v "/srv/newsblur/docker/volumes/db_mongo:/data/db" mongo:4.0 mongod --repair --dbpath /data/db
 mongodump:
-	- docker exec -it db_mongo mongodump --port 29019 -d newsblur -o /data/db/mongodump
-	# - docker exec -it db_mongo cp -fr /data/db/mongodump /data/mongodump
-	# - docker exec -it db_mongo rm -fr /data/db/
+	- docker exec -it db_mongo mongodump --port 29019 -d newsblur -o /data/mongodump
+	- cp -fr docker/volumes/db_mongo/mongodump docker/volumes/mongodump
+# - docker exec -it db_mongo cp -fr /data/db/mongodump /data/mongodump
+# - docker exec -it db_mongo rm -fr /data/db/
 mongorestore:
-	- docker exec -it db_mongo mongorestore --port 29019 -d newsblur /data/mongodump/newsblur
+	- cp -fr docker/volumes/mongodump docker/volumes/db_mongo/
+	- docker exec -it db_mongo mongorestore --port 29019 -d newsblur /data/db/mongodump/newsblur
 
 # performance tests
 perf-cli:

diff --git a/apps/rss_feeds/models.py b/apps/rss_feeds/models.py
@@ -153,7 +153,7 @@ def s3_icons_key(self):
 
     @property
     def unread_cutoff(self):
-        if self.archive_subscribers > 0:
+        if self.archive_subscribers and self.archive_subscribers > 0:
             return datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD_ARCHIVE)
         if self.premium_subscribers > 0:
             return datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
@@ -170,7 +170,7 @@ def days_of_story_hashes_for_feed(cls, feed_id):
 
     @property
     def days_of_story_hashes(self):
-        if self.archive_subscribers > 0:
+        if self.archive_subscribers and self.archive_subscribers > 0:
             return settings.DAYS_OF_STORY_HASHES_ARCHIVE
         return settings.DAYS_OF_STORY_HASHES
 
@@ -1551,7 +1551,7 @@ def trim_old_stories(cls, start=0, verbose=True, dryrun=False, total=0):
             except Feed.DoesNotExist:
                 continue
             if (feed.active_subscribers <= 0 and 
-                feed.archive_subscribers <= 0 and 
+                (not feed.archive_subscribers or feed.archive_subscribers <= 0) and 
                 (not feed.last_story_date or feed.last_story_date < month_ago)):
                 months_ago = 6
                 if feed.last_story_date:
@@ -1575,7 +1575,7 @@ def story_cutoff(self):
         return self.number_of_stories_to_store()
 
     def number_of_stories_to_store(self, pre_archive=False):
-        if self.archive_subscribers > 0 and not pre_archive:
+        if self.archive_subscribers and self.archive_subscribers > 0 and not pre_archive:
             return 10000
 
         cutoff = 500
@@ -2285,7 +2285,7 @@ def get_next_scheduled_update(self, force=False, verbose=True, premium_speed=Fal
             total = max(total, 60*2)
 
         # Pro subscribers get absolute minimum
-        if self.pro_subscribers >= 1:
+        if self.pro_subscribers and self.pro_subscribers >= 1:
             if self.stories_last_month == 0:
                 total = min(total, 60)
             else:

diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py
@@ -1215,6 +1215,7 @@ def fetch_and_process_archive_pages(self, feed_id):
                                 break
                         else:
                             logging.debug('   ---> [%-30s] ~FBFeed has no more RFC5005 links...' % (feed.log_title[:30]))
+                            break
 
                         before_story_hashes = len(seen_story_hashes)
                         pfeed.process()
@@ -1223,7 +1224,6 @@ def fetch_and_process_archive_pages(self, feed_id):
 
                         if before_story_hashes == after_story_hashes:
                             logging.debug('   ---> [%-30s] ~FRNo change in story hashes, but has archive link: %s' % (feed.log_title[:30], link_prev_archive))
-                            break
 
                 failed_color = "~FR" if not link_prev_archive else ""
                 logging.debug(f"   ---> [{feed.log_title[:30]:<30}] ~FGStory hashes found, archive RFC5005 ~SB{link_prev_archive}~SN: ~SB~FG{failed_color}{len(seen_story_hashes):,} stories~SN~FB")