Bump beautifulsoup4 from 4.11.2 to 4.12.3 (#150)

Bumps [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/) from 4.11.2 to 4.12.3.

---
updated-dependencies:
- dependency-name: beautifulsoup4
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/dependabot.yml

@@ -4,5 +4,8 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
-    time: "8:00"
+    
-  open-pull-requests-limit: 10
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly

.github/workflows/black.yml

@@ -0,0 +1,10 @@
+name: Black 
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/black@stable

.github/workflows/codeql-analysis.yml

@@ -13,7 +13,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 2
 
@@ -21,7 +21,7 @@ jobs:
       if: ${{ github.event_name == 'pull_request' }}
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.github/workflows/pypi.yml

@@ -7,9 +7,9 @@ jobs:
     if: startsWith(github.ref, 'refs/tags')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - name: Install pypa/build

.github/workflows/tests.yml

@@ -5,21 +5,32 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [
+          "3.6",
+          "3.7",
+          "3.8",
+          "3.9",
+          "3.10",
+          ]
+      fail-fast: false
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
-          pip install poetry==1.1.6
+          pip install poetry==1.1.13
           poetry install
+
       - name: Run pre-commit
         run: |
           git diff --name-only $TRAVIS_COMMIT_RANGE | xargs poetry run pre-commit run --files
+
       - name: Test
         run: poetry run make ci

MANIFEST.in

@@ -1,4 +1,2 @@
-include rfd/VERSION
-
 recursive-exclude * __pycache__
 recursive-include tests *

README.md

@@ -12,8 +12,9 @@
   - [View Hot Deals](#view-hot-deals)
   - [View and Sort Hot Deals](#view-and-sort-hot-deals)
   - [Search](#search)
-    - [Advanced](#advanced)
+    - [Regex](#regex)
   - [View Posts](#view-posts)
+  - [JSON Output](#json-output)
 - [Shell Completion](#shell-completion)
   - [bash](#bash)
   - [zsh](#zsh)
@@ -31,14 +32,21 @@ It is often faster to use a CLI than to load up a web page and navigate web elem
 
 ## Installation
 
-> :information_source: python2 may still work but it is [no longer supported](https://www.python.org/doc/sunset-python-2/).
+### pip
 
-```bash
+```sh
-python3 -m pip install --user rfd
+pip3 install --user rfd
 ```
 
-After installation, if you cannot access `rfd` in your path, try `python3 -m rfd`.
+This can also be installed with [pipx](https://github.com/pypa/pipx).
 
+### brew
+
+If you have [brew](https://brew.sh):
+
+```sh
+brew install davegallant/public/rfd
+```
 
 ## Usage
 
@@ -85,7 +93,7 @@ rfd threads --sort-by views --pages 10
 rfd search 'pizza'
 ```
 
-#### Advanced
+#### Regex
 
 Regular expressions can be used for search.
 
@@ -103,6 +111,16 @@ rfd posts https://forums.redflagdeals.com/kobo-vs-kindle-2396227/
 
 This allows for easy grepping and searching for desired expressions.
 
+### JSON Output
+
+All commands support JSON output.
+
+For example:
+
+```sh
+rfd threads --output json
+```
+
 ## Shell Completion
 
 Shell completion can be enabled if using `bash` or `zsh`.

VERSION

@@ -1 +0,0 @@
-rfd/VERSION

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rfd"
-version = "0.7.1"
+version = "0.9.0"
 description = "view RedFlagDeals.com from the command line"
 authors = ["Dave Gallant <davegallant@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -8,17 +8,17 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.6"
-beautifulsoup4 = "<4.9.4"
+beautifulsoup4 = ">=4.10.1"
 click = ">=7.0"
 colorama = ">=0.4.3"
 requests = ">=2.22.0"
 soupsieve = "<3.0"
 
 [tool.poetry.dev-dependencies]
-pre-commit = "1.21.0"
+pre-commit = "2.1.1"
-pylint = "^2.8.2"
+pylint = "^2.12.0"
 pytest = ">=4.6.6"
-rope = "0.19.0"
+rope = "1.1.1"
 
 [tool.poetry.scripts]
 rfd = "rfd.__main__:cli"

rfd/VERSION

@@ -1 +0,0 @@
-0.7.1

rfd/__version__.py

@@ -1,12 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from os.path import dirname, abspath, join
-
-
-def load_version():
-    with open(join(dirname(abspath(__file__)), "VERSION")) as handle:
-        return handle.read()
-
-
-version = load_version()

rfd/api.py

@@ -27,7 +27,7 @@ def get_safe_per_page(limit):
 
 def create_user_map(users):
     """Create a map of user ids to usernames."""
-    m = dict()
+    m = {}
     for user in users:
         m[user.get("user_id")] = user.get("username")
     return m
@@ -47,12 +47,12 @@ def get_threads(forum_id, pages):
     try:
         for page in range(1, pages + 1):
             response = requests.get(
-                "{}/api/topics?forum_id={}&per_page=40&page={}".format(
+                f"{API_BASE_URL}/api/topics?forum_id={forum_id}&per_page=40&page={page}"
-                    API_BASE_URL, forum_id, page
-                )
             )
             if response.status_code != 200:
-                raise Exception("When collecting threads, received a status code: %s" % response.status_code)
+                raise Exception(
+                    f"When collecting threads, received a status code: {response.status_code}"
+                )
             threads += response.json().get("topics")
     except JSONDecodeError as err:
         logging.error("Unable to decode threads. %s", err)
@@ -65,7 +65,7 @@ def get_posts(post):
     Args:
         post (str): either full url or postid
 
-    Yields:
+    Returns:
         list(Post): Posts
     """
     if is_valid_url(post):
@@ -76,29 +76,33 @@ def get_posts(post):
         raise ValueError()
 
     response = requests.get(
-        "{}/api/topics/{}/posts?per_page=40&page=1".format(API_BASE_URL, post_id)
+        f"{API_BASE_URL}/api/topics/{post_id}/posts?per_page=40&page=1"
     )
 
     total_pages = response.json().get("pager").get("total_pages")
 
+    posts = []
+
     for page in range(0, total_pages + 1):
         response = requests.get(
-            "{}/api/topics/{}/posts?per_page={}&page={}".format(
+            f"{API_BASE_URL}/api/topics/{post_id}/posts?per_page=40&page={page}"
-                API_BASE_URL, post_id, 40, page
-            )
         )
         users = create_user_map(response.json().get("users"))
 
-        posts = response.json().get("posts")
+        current_posts = response.json().get("posts")
 
-        for i in posts:
+        for _post in current_posts:
             # Sometimes votes is null
-            if i.get("votes") is not None:
+            if _post.get("votes") is not None:
-                calculated_score = calculate_score(i)
+                calculated_score = calculate_score(_post)
             else:
                 calculated_score = 0
-            yield Post(
+            posts.append(
-                body=strip_html(i.get("body")),
+                Post(
-                score=calculated_score,
+                    body=strip_html(_post.get("body")),
-                user=users[i.get("author_id")],
+                    score=calculated_score,
+                    user=users[_post.get("author_id")],
+                )
             )
+
+    return posts

rfd/cli.py

@@ -3,12 +3,24 @@ from __future__ import unicode_literals
 
 import logging
 import sys
+import json
 import click
+
+try:
+    from importlib import metadata
+except ImportError:  # for Python<3.8
+    import importlib_metadata as metadata
 from colorama import init
 from .api import get_threads, get_posts
-from .threads import parse_threads, search_threads, sort_threads, generate_thread_output
+from .threads import (
-from .posts import generate_posts_output
+    parse_threads,
-from .__version__ import version as current_version
+    search_threads,
+    sort_threads,
+    generate_thread_output,
+    ThreadEncoder,
+)
+from .posts import generate_posts_output, PostEncoder
+
 
 init()
 
@@ -18,12 +30,13 @@ logging.getLogger().addHandler(logging.StreamHandler())
 
 
 def get_version():
-    return "rfd v" + current_version
+    return "rfd v" + metadata.version("rfd")
 
-def print_version(ctx, value):
+
+def print_version(ctx, _, value):
     if not value or ctx.resilient_parsing:
         return
-    click.echo(get_version(), nl=False)
+    click.echo(get_version(), nl=True)
     ctx.exit()
 
 
@@ -45,7 +58,10 @@ def cli(ctx):
 
 @cli.command(short_help="Display all posts in a thread.")
 @click.argument("post_id")
-def posts(post_id):
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
+def posts(post_id, output):
     """Iterate all pages and display all posts in a thread.
 
     post_id can be a full url or post id only
@@ -57,12 +73,23 @@ def posts(post_id):
     """
 
     try:
-        click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
+        if output == "json":
+            click.echo_via_pager(
+                json.dumps(
+                    get_posts(post=post_id),
+                    cls=PostEncoder,
+                    indent=2,
+                    sort_keys=True,
+                )
+            )
+        else:
+            click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
+
     except ValueError:
         click.echo("Invalid post id.")
         sys.exit(1)
-    except AttributeError:
+    except AttributeError as err:
-        click.echo("The RFD API did not return the expected data.")
+        click.echo("The RFD API did not return the expected data. %s", err)
         sys.exit(1)
 
 
@@ -70,7 +97,10 @@ def posts(post_id):
 @click.option("--forum-id", default=9, help="The forum id number")
 @click.option("--pages", default=1, help="Number of pages to show. Defaults to 1.")
 @click.option("--sort-by", default=None, help="Sort threads by")
-def threads(forum_id, pages, sort_by):
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
+def threads(forum_id, pages, sort_by, output):
     """Display threads in the specified forum id. Defaults to 9 (hot deals).
 
     Popular forum ids:
@@ -87,8 +117,21 @@ def threads(forum_id, pages, sort_by):
     74 \t shopping discussion
     88 \t cell phones
     """
-    _threads = sort_threads(parse_threads(get_threads(forum_id, pages)), sort_by=sort_by)
+    _threads = sort_threads(
-    click.echo_via_pager(generate_thread_output(_threads))
+        parse_threads(get_threads(forum_id, pages)), sort_by=sort_by
+    )
+    if output == "json":
+
+        click.echo_via_pager(
+            json.dumps(
+                sort_threads(_threads, sort_by=sort_by),
+                cls=ThreadEncoder,
+                indent=2,
+                sort_keys=True,
+            )
+        )
+    else:
+        click.echo_via_pager(generate_thread_output(_threads))
 
 
 @cli.command(short_help="Search deals based on a regular expression.")
@@ -97,8 +140,11 @@ def threads(forum_id, pages, sort_by):
     "--forum-id", default=9, help="The forum id number. Defaults to 9 (hot deals)."
 )
 @click.option("--sort-by", default=None, help="Sort threads by")
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
 @click.argument("regex")
-def search(pages, forum_id, sort_by, regex):
+def search(pages, forum_id, sort_by, output, regex):
     """Search deals based on regex.
 
     Popular forum ids:
@@ -121,6 +167,17 @@ def search(pages, forum_id, sort_by, regex):
     _threads = parse_threads(get_threads(forum_id, pages=pages))
     for thread in search_threads(threads=_threads, regex=regex):
         matched_threads.append(thread)
-    click.echo_via_pager(
+
-        generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
+    if output == "json":
-    )
+        click.echo_via_pager(
+            json.dumps(
+                sort_threads(matched_threads, sort_by=sort_by),
+                indent=2,
+                sort_keys=True,
+                cls=ThreadEncoder,
+            )
+        )
+    else:
+        click.echo_via_pager(
+            generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
+        )

rfd/posts.py

@@ -1,21 +1,36 @@
 # pylint: disable=old-style-class
 import os
+import json
 from colorama import Fore, Style
 from .scores import get_vote_color
 
+
 class Post:
     def __init__(self, body, score, user):
         self.body = body
         self.score = score
         self.user = user
 
+
+class PostEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Post):
+            return dict(
+                body=o.body,
+                score=o.score,
+                user=o.user,
+            )
+        return json.JSONEncoder.default(self, o)
+
+
 def get_terminal_width():
     _, columns = os.popen("stty size", "r").read().split()
     return int(columns)
 
+
 def generate_posts_output(posts):
     output = ""
-    output += ("-" * get_terminal_width())
+    output += "-" * get_terminal_width()
     for post in posts:
         output += (
             " -"
@@ -23,10 +38,10 @@ def generate_posts_output(posts):
             + Fore.RESET
             + post.body
             + Fore.YELLOW
-            + " ({})".format(post.user)
+            + f" ({post.user})"
         )
-        output += (Style.RESET_ALL)
+        output += Style.RESET_ALL
         output += "\n"
-        output += ("-" * get_terminal_width())
+        output += "-" * get_terminal_width()
         output += "\n"
     return output

rfd/threads.py

@@ -1,9 +1,10 @@
 import re
+import json
 from colorama import Fore, Style
 from . import API_BASE_URL
 from .scores import calculate_score, get_vote_color
 
-# pylint: disable=old-style-class
+
 class Thread:
     def __init__(self, title, dealer_name, score, url, views):
         self.dealer_name = dealer_name
@@ -13,11 +14,24 @@ class Thread:
         self.views = views
 
     def __repr__(self):
-        return "Thread(%s)" % self.title
+        return f"Thread({self.title})"
+
+
+class ThreadEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Thread):
+            return dict(
+                dealer_name=o.dealer_name,
+                score=o.score,
+                title=o.title,
+                url=o.url,
+                views=o.views,
+            )
+        return json.JSONEncoder.default(self, o)
 
 
 def build_web_path(slug):
-    return "{}{}".format(API_BASE_URL, slug)
+    return f"{API_BASE_URL}{slug}"
 
 
 def get_dealer(topic):
@@ -88,12 +102,12 @@ def generate_thread_output(threads):
             + "."
             + get_vote_color(thread.score)
             + Fore.RESET
-            + "%s%s" % (dealer, thread.title)
+            + f"{dealer}{thread.title}"
             + Fore.LIGHTYELLOW_EX
-            + " (%d views)" % thread.views
+            + f" ({thread.views} views)"
             + Fore.RESET
         )
-        output += Fore.BLUE + " {}".format(thread.url)
+        output += Fore.BLUE + f" {thread.url}"
         output += Style.RESET_ALL
         output += "\n\n"
         yield output