Merge pull request #2272 from liberapay/various

liberapay/constants.py

@@ -369,8 +369,6 @@ def __missing__(self, currency):
     'make_team': (5, 60*60*24*7),  # 5 per week
     'payin.from-user': (15, 60*60*24*7),  # 15 per week
     'payin.from-ip-addr': (15, 60*60*24*7),  # 15 per week
-    'refetch_elsewhere_data': (1, 60*60*24*7),  # retry after one week
-    'refetch_repos': (1, 60*60*24),  # retry after one day
     'sign-up.email': (1, 5*60),  # this is used to detect near-simultaneous requests,
                                  # so 5 minutes should be plenty enough
     'sign-up.ip-addr': (5, 60*60),  # 5 per hour per IP address

liberapay/models/account_elsewhere.py

@@ -9,7 +9,7 @@
 from postgres.orm import Model
 from psycopg2 import IntegrityError
 
-from ..constants import AVATAR_QUERY, DOMAIN_RE, RATE_LIMITS, SUMMARY_MAX_SIZE
+from ..constants import AVATAR_QUERY, DOMAIN_RE, SUMMARY_MAX_SIZE
 from ..cron import logger
 from ..elsewhere._base import (
     ElsewhereError, InvalidServerResponse, UserNotFound,
@@ -328,12 +328,13 @@ def refresh_user_info(self):
             info = platform.get_user_info(self.domain, type_of_id, id_value, uncertain=False)
         except (InvalidServerResponse, UserNotFound) as e:
             if not self.missing_since:
-                self.db.run("""
+                self.set_attributes(missing_since=self.db.one("""
                     UPDATE elsewhere
                        SET missing_since = current_timestamp
                      WHERE id = %s
                        AND missing_since IS NULL
-                """, (self.id,))
+                 RETURNING missing_since
+                """, (self.id,)))
             raise UnableToRefreshAccount(f"{e.__class__.__name__}: {e}")
         if info.user_id is None:
             raise UnableToRefreshAccount("user_id is None")
@@ -386,33 +387,33 @@ def get_account_elsewhere(website, state, api_lookup=True):
 
 def refetch_elsewhere_data():
     # Note: the rate_limiting table is used to avoid blocking on errors
-    rl_prefix = 'refetch_elsewhere_data'
-    rl_cap, rl_period = RATE_LIMITS[rl_prefix]
     account = website.db.one("""
-        SELECT (e, p)::elsewhere_with_participant
-          FROM elsewhere e
-          JOIN participants p ON p.id = e.participant
-         WHERE e.info_fetched_at < now() - interval '90 days'
-           AND (e.missing_since IS NULL OR e.missing_since > (current_timestamp - interval '30 days'))
-           AND (p.status = 'active' OR p.receiving > 0)
-           AND e.platform NOT IN ('facebook', 'google', 'youtube')
-           AND check_rate_limit(%s || e.id::text, %s, %s)
-      ORDER BY e.info_fetched_at ASC
-         LIMIT 1
-    """, (rl_prefix + ':', rl_cap, rl_period))
+        WITH row AS (
+            SELECT e, p
+              FROM elsewhere e
+              JOIN participants p ON p.id = e.participant
+             WHERE e.info_fetched_at < now() - interval '90 days'
+               AND (e.missing_since IS NULL OR e.missing_since > (current_timestamp - interval '30 days'))
+               AND (e.last_fetch_attempt IS NULL OR e.last_fetch_attempt < (current_timestamp - interval '3 days'))
+               AND (p.status = 'active' OR p.receiving > 0)
+               AND e.platform NOT IN ('facebook', 'google', 'youtube')
+          ORDER BY e.info_fetched_at ASC
+             LIMIT 1
+        )
+        UPDATE elsewhere
+           SET last_fetch_attempt = current_timestamp
+         WHERE id = (SELECT (row.e).id FROM row)
+     RETURNING (SELECT (row.e, row.p)::elsewhere_with_participant FROM row)
+    """)
     if not account:
         return
-    rl_key = str(account.id)
-    website.db.hit_rate_limit(rl_prefix, rl_key)
     logger.debug("Refetching data of %r" % account)
     try:
         account2 = account.refresh_user_info()
     except (ElsewhereError, UnableToRefreshAccount) as e:
         logger.debug(f"The refetch failed: {e.__class__.__name__}: {e}")
         return
     if account2.id != account.id:
-        raise UnableToRefreshAccount(f"IDs don't match: {account2.id} != {account.id}")
+        raise AssertionError(f"IDs don't match: {account2.id} != {account.id}")
     if account2.info_fetched_at < (utcnow() - timedelta(days=90)):
-        raise UnableToRefreshAccount("info_fetched_at is still far in the past")
-    # The update was successful, clean up the rate_limiting table
-    website.db.run("DELETE FROM rate_limiting WHERE key = %s", (rl_prefix + ':' + rl_key,))
+        raise AssertionError("info_fetched_at is still far in the past")

liberapay/models/participant.py

@@ -3371,7 +3371,7 @@ def get_accounts_elsewhere(self, platform=None, is_team=None, url_required=False
         """, (self.id, platform, is_team))
         accounts.sort(key=lambda a: (website.platforms[a.platform].rank, a.is_team, a.user_id))
         if url_required:
-            accounts = [a for a in accounts if a.platform_data.account_url]
+            accounts = [a for a in accounts if a.platform_data.account_url and a.missing_since is None]
         return accounts
 
     def take_over(self, account, have_confirmation=False):

liberapay/models/repository.py

@@ -3,7 +3,6 @@
 from oauthlib.oauth2 import InvalidGrantError, TokenExpiredError
 from postgres.orm import Model
 
-from liberapay.constants import RATE_LIMITS
 from liberapay.cron import logger
 from liberapay.elsewhere._base import ElsewhereError
 from liberapay.models.account_elsewhere import UnableToRefreshAccount
@@ -71,34 +70,37 @@ def upsert_repos(cursor, repos, participant, info_fetched_at):
 
 def refetch_repos():
     # Note: the rate_limiting table is used to avoid blocking on errors
-    rl_prefix = 'refetch_repos'
-    rl_cap, rl_period = RATE_LIMITS[rl_prefix]
     repo = website.db.one("""
-        SELECT r.participant, r.platform
-          FROM repositories r
-         WHERE r.info_fetched_at < now() - interval '6 days'
-           AND r.participant IS NOT NULL
-           AND r.show_on_profile
-           AND check_rate_limit(%s || r.participant::text || ':' || r.platform, %s, %s)
-      ORDER BY r.info_fetched_at ASC
-         LIMIT 1
-    """, (rl_prefix + ':', rl_cap, rl_period))
+        WITH repo AS (
+            SELECT r.*
+              FROM repositories r
+             WHERE r.info_fetched_at < now() - interval '6 days'
+               AND (r.last_fetch_attempt IS NULL OR r.last_fetch_attempt < (current_timestamp - interval '1 day'))
+               AND r.participant IS NOT NULL
+               AND r.show_on_profile
+          ORDER BY r.info_fetched_at ASC
+             LIMIT 1
+        )
+        UPDATE repositories
+           SET last_fetch_attempt = current_timestamp
+         WHERE participant = (SELECT repo.participant FROM repo)
+           AND platform = (SELECT repo.platform FROM repo)
+     RETURNING participant, platform
+    """)
     if not repo:
         return
 
-    rl_key = '%s:%s' % (repo.participant, repo.platform)
-    website.db.hit_rate_limit(rl_prefix, rl_key)
     participant = Participant.from_id(repo.participant)
     accounts = participant.get_accounts_elsewhere(repo.platform)
     if not accounts:
         return
     for account in accounts:
         if account.missing_since is not None:
             continue
-        _refetch_repos_for_account(rl_prefix, rl_key, participant, account)
+        _refetch_repos_for_account(participant, account)
 
 
-def _refetch_repos_for_account(rl_prefix, rl_key, participant, account):
+def _refetch_repos_for_account(participant, account):
     sess = account.get_auth_session()
     logger.debug(
         "Refetching profile data for participant ~%s from %s account %s" %
@@ -148,6 +150,3 @@ def _refetch_repos_for_account(rl_prefix, rl_key, participant, account):
     except (InvalidGrantError, TokenExpiredError) as e:
         logger.debug("The refetch failed: %s" % e)
         return
-
-    # The update was successful, clean up the rate_limiting table
-    website.db.run("DELETE FROM rate_limiting WHERE key = %s", (rl_prefix + ':' + rl_key,))

liberapay/wireup.py

@@ -581,6 +581,7 @@ def accounts_elsewhere(app_conf, asset, canonical_url, db):
               JOIN participants p ON p.id = e.participant
              WHERE p.status = 'active'
                AND p.hide_from_lists = 0
+               AND e.missing_since IS NULL
           GROUP BY e.platform
                ) a
       ORDER BY c DESC, platform ASC

sql/branch.sql

@@ -0,0 +1,5 @@
+BEGIN;
+    ALTER TABLE elsewhere ADD COLUMN last_fetch_attempt timestamptz;
+    ALTER TABLE repositories ADD COLUMN last_fetch_attempt timestamptz;
+    DELETE FROM rate_limiting WHERE key LIKE 'refetch_%';
+END;

www/explore/elsewhere/%platform.spt

@@ -10,6 +10,7 @@ platforms = website.db.all("""
           FROM elsewhere e
           JOIN participants p ON p.id = e.participant
          WHERE e.platform IN %s
+           AND e.missing_since IS NULL
            AND p.status = 'active'
            AND p.hide_from_lists = 0
       GROUP BY e.platform
@@ -52,6 +53,7 @@ if platform:
              WHERE p.status = 'active'
                AND p.hide_from_lists = 0
                AND e.platform = %s
+               AND e.missing_since IS NULL
                AND p.id < %s
           ORDER BY (CASE WHEN %s THEN random() ELSE 0 END), p.id DESC
              LIMIT %s

www/explore/pledges.spt

@@ -9,6 +9,7 @@ pledgees = website.db.all("""
      WHERE p.status = 'stub'
        AND p.receiving > 0
        AND p.hide_from_lists = 0
+       AND e.missing_since IS NULL
   ORDER BY p.npatrons DESC, convert(p.receiving, 'EUR') DESC, e.id DESC
      LIMIT %s
     OFFSET %s

www/explore/repositories.spt

@@ -9,6 +9,7 @@ repos = website.db.all("""
       JOIN participants p ON p.id = e.participant
      WHERE r.stars_count > 1
        AND r.show_on_profile
+       AND e.missing_since IS NULL
        AND p.status = 'active'
   ORDER BY r.stars_count DESC, r.id DESC
      LIMIT %s

www/payment-providers/%provider/connect.spt

@@ -147,6 +147,23 @@ elif 'state' in request.qs:
     else:
         raise ValueError(provider_name)
 
+    # Check that this payment account isn't connected to another Liberapay account
+    # marked as fraudulent
+    is_linked_to_fraud = website.db.one("""
+        SELECT true
+          FROM payment_accounts a
+          JOIN participants p ON p.id = a.participant
+         WHERE a.id = %(account_id)s
+           AND p.is_suspended IS TRUE
+         LIMIT 1
+    """, account_data)
+    if is_linked_to_fraud:
+        raise response.error(403, _(
+            "The {provider} account you are attempting to connect is linked to "
+            "another Liberapay account marked as fraudulent.",
+            provider=provider.display_name,
+        ))
+
     # Upsert the account in our database
     account_data['token'] = json.dumps(account_data['token'])
     with website.db.get_cursor() as cursor: