Bump beautifulsoup4 from 4.11.2 to 4.12.3 (#150)

Bumps [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/) from 4.11.2 to 4.12.3.

---
updated-dependencies:
- dependency-name: beautifulsoup4
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/dependabot.yml

@@ -4,5 +4,8 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
-    time: "8:00"
-  open-pull-requests-limit: 10
+    
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly

.github/workflows/black.yml

@@ -6,5 +6,5 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: psf/black@stable

.github/workflows/codeql-analysis.yml

@@ -13,7 +13,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 2
 
@@ -21,7 +21,7 @@ jobs:
       if: ${{ github.event_name == 'pull_request' }}
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.github/workflows/pypi.yml

@@ -7,9 +7,9 @@ jobs:
     if: startsWith(github.ref, 'refs/tags')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - name: Install pypa/build

.github/workflows/tests.yml

@@ -5,21 +5,32 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [
+          "3.6",
+          "3.7",
+          "3.8",
+          "3.9",
+          "3.10",
+          ]
+      fail-fast: false
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
-          pip install poetry==1.1.6
+          pip install poetry==1.1.13
           poetry install
+
       - name: Run pre-commit
         run: |
           git diff --name-only $TRAVIS_COMMIT_RANGE | xargs poetry run pre-commit run --files
+
       - name: Test
         run: poetry run make ci

MANIFEST.in

@@ -1,4 +1,2 @@
-include rfd/VERSION
-
 recursive-exclude * __pycache__
 recursive-include tests *

README.md

@@ -12,8 +12,9 @@
   - [View Hot Deals](#view-hot-deals)
   - [View and Sort Hot Deals](#view-and-sort-hot-deals)
   - [Search](#search)
-    - [Advanced](#advanced)
+    - [Regex](#regex)
   - [View Posts](#view-posts)
+  - [JSON Output](#json-output)
 - [Shell Completion](#shell-completion)
   - [bash](#bash)
   - [zsh](#zsh)
@@ -47,7 +48,6 @@ If you have [brew](https://brew.sh):
 brew install davegallant/public/rfd
 ```
 
-
 ## Usage
 
 All commands open up in a [terminal pager](https://en.wikipedia.org/wiki/Terminal_pager).
@@ -93,7 +93,7 @@ rfd threads --sort-by views --pages 10
 rfd search 'pizza'
 ```
 
-#### Advanced
+#### Regex
 
 Regular expressions can be used for search.
 
@@ -111,6 +111,16 @@ rfd posts https://forums.redflagdeals.com/kobo-vs-kindle-2396227/
 
 This allows for easy grepping and searching for desired expressions.
 
+### JSON Output
+
+All commands support JSON output.
+
+For example:
+
+```sh
+rfd threads --output json
+```
+
 ## Shell Completion
 
 Shell completion can be enabled if using `bash` or `zsh`.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rfd"
-version = "0.8.0"
+version = "0.9.0"
 description = "view RedFlagDeals.com from the command line"
 authors = ["Dave Gallant <davegallant@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -8,17 +8,17 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.6"
-beautifulsoup4 = "<4.10.1"
+beautifulsoup4 = ">=4.10.1"
 click = ">=7.0"
 colorama = ">=0.4.3"
 requests = ">=2.22.0"
 soupsieve = "<3.0"
 
 [tool.poetry.dev-dependencies]
-pre-commit = "1.21.0"
-pylint = "^2.11.1"
+pre-commit = "2.1.1"
+pylint = "^2.12.0"
 pytest = ">=4.6.6"
-rope = "0.20.1"
+rope = "1.1.1"
 
 [tool.poetry.scripts]
 rfd = "rfd.__main__:cli"

rfd/api.py

@@ -65,7 +65,7 @@ def get_posts(post):
     Args:
         post (str): either full url or postid
 
-    Yields:
+    Returns:
         list(Post): Posts
     """
     if is_valid_url(post):
@@ -81,22 +81,28 @@ def get_posts(post):
 
     total_pages = response.json().get("pager").get("total_pages")
 
+    posts = []
+
     for page in range(0, total_pages + 1):
         response = requests.get(
             f"{API_BASE_URL}/api/topics/{post_id}/posts?per_page=40&page={page}"
         )
         users = create_user_map(response.json().get("users"))
 
-        posts = response.json().get("posts")
+        current_posts = response.json().get("posts")
 
-        for i in posts:
+        for _post in current_posts:
             # Sometimes votes is null
-            if i.get("votes") is not None:
-                calculated_score = calculate_score(i)
+            if _post.get("votes") is not None:
+                calculated_score = calculate_score(_post)
             else:
                 calculated_score = 0
-            yield Post(
-                body=strip_html(i.get("body")),
-                score=calculated_score,
-                user=users[i.get("author_id")],
+            posts.append(
+                Post(
+                    body=strip_html(_post.get("body")),
+                    score=calculated_score,
+                    user=users[_post.get("author_id")],
+                )
             )
+
+    return posts

rfd/cli.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import logging
 import sys
+import json
 import click
 
 try:
@@ -11,8 +12,14 @@ except ImportError:  # for Python<3.8
     import importlib_metadata as metadata
 from colorama import init
 from .api import get_threads, get_posts
-from .threads import parse_threads, search_threads, sort_threads, generate_thread_output
-from .posts import generate_posts_output
+from .threads import (
+    parse_threads,
+    search_threads,
+    sort_threads,
+    generate_thread_output,
+    ThreadEncoder,
+)
+from .posts import generate_posts_output, PostEncoder
 
 
 init()
@@ -51,7 +58,10 @@ def cli(ctx):
 
 @cli.command(short_help="Display all posts in a thread.")
 @click.argument("post_id")
-def posts(post_id):
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
+def posts(post_id, output):
     """Iterate all pages and display all posts in a thread.
 
     post_id can be a full url or post id only
@@ -63,12 +73,23 @@ def posts(post_id):
     """
 
     try:
-        click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
+        if output == "json":
+            click.echo_via_pager(
+                json.dumps(
+                    get_posts(post=post_id),
+                    cls=PostEncoder,
+                    indent=2,
+                    sort_keys=True,
+                )
+            )
+        else:
+            click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
+
     except ValueError:
         click.echo("Invalid post id.")
         sys.exit(1)
-    except AttributeError:
-        click.echo("The RFD API did not return the expected data.")
+    except AttributeError as err:
+        click.echo("The RFD API did not return the expected data. %s", err)
         sys.exit(1)
 
 
@@ -76,7 +97,10 @@ def posts(post_id):
 @click.option("--forum-id", default=9, help="The forum id number")
 @click.option("--pages", default=1, help="Number of pages to show. Defaults to 1.")
 @click.option("--sort-by", default=None, help="Sort threads by")
-def threads(forum_id, pages, sort_by):
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
+def threads(forum_id, pages, sort_by, output):
     """Display threads in the specified forum id. Defaults to 9 (hot deals).
 
     Popular forum ids:
@@ -96,7 +120,18 @@ def threads(forum_id, pages, sort_by):
     _threads = sort_threads(
         parse_threads(get_threads(forum_id, pages)), sort_by=sort_by
     )
-    click.echo_via_pager(generate_thread_output(_threads))
+    if output == "json":
+
+        click.echo_via_pager(
+            json.dumps(
+                sort_threads(_threads, sort_by=sort_by),
+                cls=ThreadEncoder,
+                indent=2,
+                sort_keys=True,
+            )
+        )
+    else:
+        click.echo_via_pager(generate_thread_output(_threads))
 
 
 @cli.command(short_help="Search deals based on a regular expression.")
@@ -105,8 +140,11 @@ def threads(forum_id, pages, sort_by):
     "--forum-id", default=9, help="The forum id number. Defaults to 9 (hot deals)."
 )
 @click.option("--sort-by", default=None, help="Sort threads by")
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
 @click.argument("regex")
-def search(pages, forum_id, sort_by, regex):
+def search(pages, forum_id, sort_by, output, regex):
     """Search deals based on regex.
 
     Popular forum ids:
@@ -129,6 +167,17 @@ def search(pages, forum_id, sort_by, regex):
     _threads = parse_threads(get_threads(forum_id, pages=pages))
     for thread in search_threads(threads=_threads, regex=regex):
         matched_threads.append(thread)
-    click.echo_via_pager(
-        generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
-    )
+
+    if output == "json":
+        click.echo_via_pager(
+            json.dumps(
+                sort_threads(matched_threads, sort_by=sort_by),
+                indent=2,
+                sort_keys=True,
+                cls=ThreadEncoder,
+            )
+        )
+    else:
+        click.echo_via_pager(
+            generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
+        )

rfd/posts.py

@@ -1,5 +1,6 @@
 # pylint: disable=old-style-class
 import os
+import json
 from colorama import Fore, Style
 from .scores import get_vote_color
 
@@ -11,6 +12,17 @@ class Post:
         self.user = user
 
 
+class PostEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Post):
+            return dict(
+                body=o.body,
+                score=o.score,
+                user=o.user,
+            )
+        return json.JSONEncoder.default(self, o)
+
+
 def get_terminal_width():
     _, columns = os.popen("stty size", "r").read().split()
     return int(columns)

rfd/threads.py

@@ -1,4 +1,5 @@
 import re
+import json
 from colorama import Fore, Style
 from . import API_BASE_URL
 from .scores import calculate_score, get_vote_color
@@ -16,6 +17,19 @@ class Thread:
         return f"Thread({self.title})"
 
 
+class ThreadEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Thread):
+            return dict(
+                dealer_name=o.dealer_name,
+                score=o.score,
+                title=o.title,
+                url=o.url,
+                views=o.views,
+            )
+        return json.JSONEncoder.default(self, o)
+
+
 def build_web_path(slug):
     return f"{API_BASE_URL}{slug}"