Bump beautifulsoup4 from 4.11.2 to 4.12.3 (#150)

Bumps [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/) from 4.11.2 to 4.12.3.

---
updated-dependencies:
- dependency-name: beautifulsoup4
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/dependabot.yml

@@ -4,5 +4,8 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
-    time: "8:00"
-  open-pull-requests-limit: 10
+    
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly

.github/workflows/black.yml

@@ -0,0 +1,10 @@
+name: Black 
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/black@stable

.github/workflows/codeql-analysis.yml

@@ -13,7 +13,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 2
 
@@ -21,7 +21,7 @@ jobs:
       if: ${{ github.event_name == 'pull_request' }}
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.github/workflows/pypi.yml

@@ -7,9 +7,9 @@ jobs:
     if: startsWith(github.ref, 'refs/tags')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - name: Install pypa/build

.github/workflows/tests.yml

@@ -5,21 +5,32 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [
+          "3.6",
+          "3.7",
+          "3.8",
+          "3.9",
+          "3.10",
+          ]
+      fail-fast: false
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
-          pip install poetry==1.1.6
+          pip install poetry==1.1.13
           poetry install
+
       - name: Run pre-commit
         run: |
           git diff --name-only $TRAVIS_COMMIT_RANGE | xargs poetry run pre-commit run --files
+
       - name: Test
         run: poetry run make ci

MANIFEST.in

@@ -1,4 +1,2 @@
-include rfd/VERSION
-
 recursive-exclude * __pycache__
 recursive-include tests *

README.md

@@ -7,13 +7,14 @@
 <!-- BEGIN mktoc -->
 - [Description](#description)
 - [Motivation](#motivation)
-- [Install](#install)
+- [Installation](#installation)
 - [Usage](#usage)
   - [View Hot Deals](#view-hot-deals)
   - [View and Sort Hot Deals](#view-and-sort-hot-deals)
   - [Search](#search)
-    - [Advanced](#advanced)
+    - [Regex](#regex)
   - [View Posts](#view-posts)
+  - [JSON Output](#json-output)
 - [Shell Completion](#shell-completion)
   - [bash](#bash)
   - [zsh](#zsh)
@@ -21,7 +22,7 @@
 
 ## Description
 
-This is a CLI utility that allows you to view [Redflagdeals.com](https://forums.redflagdeals.com) on the command line.
+This is a CLI utility that allows you to view [RedFlagDeals.com](https://forums.redflagdeals.com) on the command line.
 
 ![screenshot](https://user-images.githubusercontent.com/4519234/85969861-e10a4100-b996-11ea-9a31-6203322c60ee.png)
 
@@ -29,10 +30,22 @@ This is a CLI utility that allows you to view [Redflagdeals.com](https://forums.
 
 It is often faster to use a CLI than to load up a web page and navigate web elements. This tool can search for deals and sort them based on score and views. It is also able to load entire threads (without pagination) for additional analysis.
 
-## Install
+## Installation
 
-```bash
-pip install rfd
+### pip
+
+```sh
+pip3 install --user rfd
+```
+
+This can also be installed with [pipx](https://github.com/pypa/pipx).
+
+### brew
+
+If you have [brew](https://brew.sh):
+
+```sh
+brew install davegallant/public/rfd
 ```
 
 ## Usage
@@ -80,7 +93,7 @@ rfd threads --sort-by views --pages 10
 rfd search 'pizza'
 ```
 
-#### Advanced
+#### Regex
 
 Regular expressions can be used for search.
 
@@ -98,6 +111,16 @@ rfd posts https://forums.redflagdeals.com/kobo-vs-kindle-2396227/
 
 This allows for easy grepping and searching for desired expressions.
 
+### JSON Output
+
+All commands support JSON output.
+
+For example:
+
+```sh
+rfd threads --output json
+```
+
 ## Shell Completion
 
 Shell completion can be enabled if using `bash` or `zsh`.

VERSION

@@ -1 +0,0 @@
-rfd/VERSION

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rfd"
-version = "0.7.0"
+version = "0.9.0"
 description = "view RedFlagDeals.com from the command line"
 authors = ["Dave Gallant <davegallant@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -8,17 +8,20 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.6"
-beautifulsoup4 = "<4.9.4"
+beautifulsoup4 = ">=4.10.1"
 click = ">=7.0"
 colorama = ">=0.4.3"
 requests = ">=2.22.0"
 soupsieve = "<3.0"
 
 [tool.poetry.dev-dependencies]
-pre-commit = "1.21.0"
-pylint = "^2.8.2"
+pre-commit = "2.1.1"
+pylint = "^2.12.0"
 pytest = ">=4.6.6"
-rope = "0.19.0"
+rope = "1.1.1"
+
+[tool.poetry.scripts]
+rfd = "rfd.__main__:cli"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

rfd/VERSION

@@ -1 +0,0 @@
-0.7.0

rfd/__version__.py

@@ -1,12 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from os.path import dirname, abspath, join
-
-
-def load_version():
-    with open(join(dirname(abspath(__file__)), "VERSION")) as handle:
-        return handle.read()
-
-
-version = load_version()

rfd/api.py

@@ -27,7 +27,7 @@ def get_safe_per_page(limit):
 
 def create_user_map(users):
     """Create a map of user ids to usernames."""
-    m = dict()
+    m = {}
     for user in users:
         m[user.get("user_id")] = user.get("username")
     return m
@@ -47,12 +47,12 @@ def get_threads(forum_id, pages):
     try:
         for page in range(1, pages + 1):
             response = requests.get(
-                "{}/api/topics?forum_id={}&per_page=40&page={}".format(
-                    API_BASE_URL, forum_id, page
-                )
+                f"{API_BASE_URL}/api/topics?forum_id={forum_id}&per_page=40&page={page}"
             )
             if response.status_code != 200:
-                raise Exception("When collecting threads, received a status code: %s" % response.status_code)
+                raise Exception(
+                    f"When collecting threads, received a status code: {response.status_code}"
+                )
             threads += response.json().get("topics")
     except JSONDecodeError as err:
         logging.error("Unable to decode threads. %s", err)
@@ -65,7 +65,7 @@ def get_posts(post):
     Args:
         post (str): either full url or postid
 
-    Yields:
+    Returns:
         list(Post): Posts
     """
     if is_valid_url(post):
@@ -76,29 +76,33 @@ def get_posts(post):
         raise ValueError()
 
     response = requests.get(
-        "{}/api/topics/{}/posts?per_page=40&page=1".format(API_BASE_URL, post_id)
+        f"{API_BASE_URL}/api/topics/{post_id}/posts?per_page=40&page=1"
     )
 
     total_pages = response.json().get("pager").get("total_pages")
 
+    posts = []
+
     for page in range(0, total_pages + 1):
         response = requests.get(
-            "{}/api/topics/{}/posts?per_page={}&page={}".format(
-                API_BASE_URL, post_id, 40, page
-            )
+            f"{API_BASE_URL}/api/topics/{post_id}/posts?per_page=40&page={page}"
         )
         users = create_user_map(response.json().get("users"))
 
-        posts = response.json().get("posts")
+        current_posts = response.json().get("posts")
 
-        for i in posts:
+        for _post in current_posts:
             # Sometimes votes is null
-            if i.get("votes") is not None:
-                calculated_score = calculate_score(i)
+            if _post.get("votes") is not None:
+                calculated_score = calculate_score(_post)
             else:
                 calculated_score = 0
-            yield Post(
-                body=strip_html(i.get("body")),
-                score=calculated_score,
-                user=users[i.get("author_id")],
+            posts.append(
+                Post(
+                    body=strip_html(_post.get("body")),
+                    score=calculated_score,
+                    user=users[_post.get("author_id")],
+                )
             )
+
+    return posts

rfd/cli.py

@@ -3,12 +3,24 @@ from __future__ import unicode_literals
 
 import logging
 import sys
+import json
 import click
+
+try:
+    from importlib import metadata
+except ImportError:  # for Python<3.8
+    import importlib_metadata as metadata
 from colorama import init
 from .api import get_threads, get_posts
-from .threads import parse_threads, search_threads, sort_threads, generate_thread_output
-from .posts import generate_posts_output
-from .__version__ import version as current_version
+from .threads import (
+    parse_threads,
+    search_threads,
+    sort_threads,
+    generate_thread_output,
+    ThreadEncoder,
+)
+from .posts import generate_posts_output, PostEncoder
+
 
 init()
 
@@ -18,12 +30,13 @@ logging.getLogger().addHandler(logging.StreamHandler())
 
 
 def get_version():
-    return "rfd v" + current_version
+    return "rfd v" + metadata.version("rfd")
 
-def print_version(ctx, value):
+
+def print_version(ctx, _, value):
     if not value or ctx.resilient_parsing:
         return
-    click.echo(get_version(), nl=False)
+    click.echo(get_version(), nl=True)
     ctx.exit()
 
 
@@ -45,7 +58,10 @@ def cli(ctx):
 
 @cli.command(short_help="Display all posts in a thread.")
 @click.argument("post_id")
-def posts(post_id):
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
+def posts(post_id, output):
     """Iterate all pages and display all posts in a thread.
 
     post_id can be a full url or post id only
@@ -57,12 +73,23 @@ def posts(post_id):
     """
 
     try:
-        click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
+        if output == "json":
+            click.echo_via_pager(
+                json.dumps(
+                    get_posts(post=post_id),
+                    cls=PostEncoder,
+                    indent=2,
+                    sort_keys=True,
+                )
+            )
+        else:
+            click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
+
     except ValueError:
         click.echo("Invalid post id.")
         sys.exit(1)
-    except AttributeError:
-        click.echo("The RFD API did not return the expected data.")
+    except AttributeError as err:
+        click.echo("The RFD API did not return the expected data. %s", err)
         sys.exit(1)
 
 
@@ -70,7 +97,10 @@ def posts(post_id):
 @click.option("--forum-id", default=9, help="The forum id number")
 @click.option("--pages", default=1, help="Number of pages to show. Defaults to 1.")
 @click.option("--sort-by", default=None, help="Sort threads by")
-def threads(forum_id, pages, sort_by):
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
+def threads(forum_id, pages, sort_by, output):
     """Display threads in the specified forum id. Defaults to 9 (hot deals).
 
     Popular forum ids:
@@ -87,8 +117,21 @@ def threads(forum_id, pages, sort_by):
     74 \t shopping discussion
     88 \t cell phones
     """
-    _threads = sort_threads(parse_threads(get_threads(forum_id, pages)), sort_by=sort_by)
-    click.echo_via_pager(generate_thread_output(_threads))
+    _threads = sort_threads(
+        parse_threads(get_threads(forum_id, pages)), sort_by=sort_by
+    )
+    if output == "json":
+
+        click.echo_via_pager(
+            json.dumps(
+                sort_threads(_threads, sort_by=sort_by),
+                cls=ThreadEncoder,
+                indent=2,
+                sort_keys=True,
+            )
+        )
+    else:
+        click.echo_via_pager(generate_thread_output(_threads))
 
 
 @cli.command(short_help="Search deals based on a regular expression.")
@@ -97,8 +140,11 @@ def threads(forum_id, pages, sort_by):
     "--forum-id", default=9, help="The forum id number. Defaults to 9 (hot deals)."
 )
 @click.option("--sort-by", default=None, help="Sort threads by")
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
 @click.argument("regex")
-def search(pages, forum_id, sort_by, regex):
+def search(pages, forum_id, sort_by, output, regex):
     """Search deals based on regex.
 
     Popular forum ids:
@@ -121,6 +167,17 @@ def search(pages, forum_id, sort_by, regex):
     _threads = parse_threads(get_threads(forum_id, pages=pages))
     for thread in search_threads(threads=_threads, regex=regex):
         matched_threads.append(thread)
-    click.echo_via_pager(
-        generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
-    )
+
+    if output == "json":
+        click.echo_via_pager(
+            json.dumps(
+                sort_threads(matched_threads, sort_by=sort_by),
+                indent=2,
+                sort_keys=True,
+                cls=ThreadEncoder,
+            )
+        )
+    else:
+        click.echo_via_pager(
+            generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
+        )

rfd/posts.py

@@ -1,21 +1,36 @@
 # pylint: disable=old-style-class
 import os
+import json
 from colorama import Fore, Style
 from .scores import get_vote_color
 
+
 class Post:
     def __init__(self, body, score, user):
         self.body = body
         self.score = score
         self.user = user
 
+
+class PostEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Post):
+            return dict(
+                body=o.body,
+                score=o.score,
+                user=o.user,
+            )
+        return json.JSONEncoder.default(self, o)
+
+
 def get_terminal_width():
     _, columns = os.popen("stty size", "r").read().split()
     return int(columns)
 
+
 def generate_posts_output(posts):
     output = ""
-    output += ("-" * get_terminal_width())
+    output += "-" * get_terminal_width()
     for post in posts:
         output += (
             " -"
@@ -23,10 +38,10 @@ def generate_posts_output(posts):
             + Fore.RESET
             + post.body
             + Fore.YELLOW
-            + " ({})".format(post.user)
+            + f" ({post.user})"
         )
-        output += (Style.RESET_ALL)
+        output += Style.RESET_ALL
         output += "\n"
-        output += ("-" * get_terminal_width())
+        output += "-" * get_terminal_width()
         output += "\n"
     return output

rfd/threads.py

@@ -1,9 +1,10 @@
 import re
+import json
 from colorama import Fore, Style
 from . import API_BASE_URL
 from .scores import calculate_score, get_vote_color
 
-# pylint: disable=old-style-class
+
 class Thread:
     def __init__(self, title, dealer_name, score, url, views):
         self.dealer_name = dealer_name
@@ -13,11 +14,24 @@ class Thread:
         self.views = views
 
     def __repr__(self):
-        return "Thread(%s)" % self.title
+        return f"Thread({self.title})"
+
+
+class ThreadEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Thread):
+            return dict(
+                dealer_name=o.dealer_name,
+                score=o.score,
+                title=o.title,
+                url=o.url,
+                views=o.views,
+            )
+        return json.JSONEncoder.default(self, o)
 
 
 def build_web_path(slug):
-    return "{}{}".format(API_BASE_URL, slug)
+    return f"{API_BASE_URL}{slug}"
 
 
 def get_dealer(topic):
@@ -88,12 +102,12 @@ def generate_thread_output(threads):
             + "."
             + get_vote_color(thread.score)
             + Fore.RESET
-            + "%s%s" % (dealer, thread.title)
+            + f"{dealer}{thread.title}"
             + Fore.LIGHTYELLOW_EX
-            + " (%d views)" % thread.views
+            + f" ({thread.views} views)"
             + Fore.RESET
         )
-        output += Fore.BLUE + " {}".format(thread.url)
+        output += Fore.BLUE + f" {thread.url}"
         output += Style.RESET_ALL
         output += "\n\n"
         yield output